From 43d378b0e3b7e349ea699cbdbac5fe5bc8969e67 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 06:03:31 +0000 Subject: [PATCH 1/4] fix(syntax): record a star's source range in a located parse A star's span was a plain Option, which a located parse never fills, so `*`, `t.*` and a star with EXCLUDE, REPLACE or RENAME had no range in any dialect. It is now a SourceSpan like other located nodes: recorded only by a located parse, ignored by equality, and left out of serialised trees when absent. Scope findings that fall back to the first located column, name or star in their scope now point at that star instead of the statement start: a star modifier naming an unknown column, a USING key one side lacks, and a duplicate CTE name in a query that projects a star. --- .../src/duckdb/profile/tests/test_checks.rs | 2 +- .../src/lineage/_helpers/checks/issues.rs | 2 +- .../src/postgres/profile/tests/test_checks.rs | 2 +- .../snowflake/profile/tests/test_statement_findings.rs | 2 +- .../src/parse/_helpers/tree/nodes.rs | 2 +- .../src/parse/_helpers/tree/spans.rs | 2 +- .../src/parse/_helpers/tree/nodes.rs | 2 +- .../src/parse/_helpers/tree/spans.rs | 2 +- .../src/parse/_helpers/tree/nodes.rs | 2 +- .../src/parse/_helpers/tree/spans.rs | 2 +- .../src/parse/_helpers/tree/nodes.rs | 2 +- .../src/parse/_helpers/tree/spans.rs | 2 +- .../src/parse/_helpers/tree/nodes.rs | 2 +- .../src/parse/_helpers/tree/spans.rs | 2 +- .../src/parser/_helpers/expressions/grouping.rs | 6 +++--- .../src/parser/_helpers/expressions/primary.rs | 6 +++--- .../src/parser/_helpers/functions/typed.rs | 2 +- .../src/parser/_helpers/queries/select.rs | 2 +- .../src/parser/_helpers/support/names.rs | 2 +- .../src/parser/_helpers/types/data_types.rs | 2 +- .../src/parse/_helpers/tree/nodes.rs | 2 +- .../src/parse/_helpers/tree/spans.rs | 2 +- .../src/parse/_helpers/statements/queries.rs | 2 +- .../src/parse/_helpers/values/primary.rs | 2 +- crates/sqlquack-syntax/src/expressions/names/models.rs | 8 +++++--- .../src/expressions/tree/_helpers/accessors.rs | 3 ++- 26 files changed, 35 insertions(+), 32 deletions(-) diff --git a/crates/sqlquack-semantics/src/duckdb/profile/tests/test_checks.rs b/crates/sqlquack-semantics/src/duckdb/profile/tests/test_checks.rs index fcaffbdee..3e0aaf3fb 100644 --- a/crates/sqlquack-semantics/src/duckdb/profile/tests/test_checks.rs +++ b/crates/sqlquack-semantics/src/duckdb/profile/tests/test_checks.rs @@ -145,7 +145,7 @@ fn given_a_duckdb_select_when_checking_its_scope_rules_then_broken_rules_are_fou expected_findings: &[( "E201", "Star modifier references unknown column 'quantity'", - 0, + 60, )], }, QueryFindingsTestCase { diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/checks/issues.rs b/crates/sqlquack-semantics/src/lineage/_helpers/checks/issues.rs index bfb3403db..4bb997c0a 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/checks/issues.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/checks/issues.rs @@ -40,7 +40,7 @@ fn own_location(node: &Expression) -> Option { match node { Expression::Column(column) => column.span.or(column.name.span), Expression::Identifier(identifier) => identifier.span, - Expression::Star(star) => star.span, + Expression::Star(star) => star.span.0, Expression::Count(count) => count.span, Expression::Function(function) => function.span.0, _ => None, diff --git a/crates/sqlquack-semantics/src/postgres/profile/tests/test_checks.rs b/crates/sqlquack-semantics/src/postgres/profile/tests/test_checks.rs index 7bf5ce5e0..98be6d61b 100644 --- a/crates/sqlquack-semantics/src/postgres/profile/tests/test_checks.rs +++ b/crates/sqlquack-semantics/src/postgres/profile/tests/test_checks.rs @@ -154,7 +154,7 @@ fn given_a_postgres_query_when_checking_types_and_names_then_findings_follow_pos QueryFindingsTestCase { description: "two CTEs of one name (18.6: 42712)", sql: "WITH c AS (SELECT 1), c AS (SELECT 2) SELECT * FROM c", - expected_findings: &[("E233", "Duplicate CTE name 'c'", 0)], + expected_findings: &[("E233", "Duplicate CTE name 'c'", 45)], }, QueryFindingsTestCase { description: "an output name inside an ORDER BY expression (18.6: 42703)", diff --git a/crates/sqlquack-semantics/src/snowflake/profile/tests/test_statement_findings.rs b/crates/sqlquack-semantics/src/snowflake/profile/tests/test_statement_findings.rs index ca036d900..b5552b3a2 100644 --- a/crates/sqlquack-semantics/src/snowflake/profile/tests/test_statement_findings.rs +++ b/crates/sqlquack-semantics/src/snowflake/profile/tests/test_statement_findings.rs @@ -87,7 +87,7 @@ fn given_a_query_when_resolving_its_names_then_unknown_ambiguous_and_unqualifiab ( "E201", "JOIN USING column 'status' must exist on both sides", - 0, + 7, ), ("E201", "Unknown UNPIVOT input column 'missing'", 103), ], diff --git a/crates/sqlquack-syntax-bigquery/src/parse/_helpers/tree/nodes.rs b/crates/sqlquack-syntax-bigquery/src/parse/_helpers/tree/nodes.rs index 3dfdf92c3..7fb62d88d 100644 --- a/crates/sqlquack-syntax-bigquery/src/parse/_helpers/tree/nodes.rs +++ b/crates/sqlquack-syntax-bigquery/src/parse/_helpers/tree/nodes.rs @@ -60,7 +60,7 @@ pub(crate) fn star(table: Option, except: Option>) - replace: None, rename: None, trailing_comments: Vec::new(), - span: None, + span: SourceSpan::default(), qualified: None, } } diff --git a/crates/sqlquack-syntax-bigquery/src/parse/_helpers/tree/spans.rs b/crates/sqlquack-syntax-bigquery/src/parse/_helpers/tree/spans.rs index b42095824..5012a265f 100644 --- a/crates/sqlquack-syntax-bigquery/src/parse/_helpers/tree/spans.rs +++ b/crates/sqlquack-syntax-bigquery/src/parse/_helpers/tree/spans.rs @@ -10,7 +10,7 @@ fn span_field(expression: &mut Expression) -> Option<&mut Option> { match expression { Expression::Column(node) => Some(&mut node.span), Expression::Identifier(node) => Some(&mut node.span), - Expression::Star(node) => Some(&mut node.span), + Expression::Star(node) => Some(&mut node.span.0), Expression::Count(node) => Some(&mut node.span), Expression::Function(node) => Some(&mut node.span.0), Expression::AggregateFunction(node) => Some(&mut node.span.0), diff --git a/crates/sqlquack-syntax-clickhouse/src/parse/_helpers/tree/nodes.rs b/crates/sqlquack-syntax-clickhouse/src/parse/_helpers/tree/nodes.rs index 561f4eedb..d55d84152 100644 --- a/crates/sqlquack-syntax-clickhouse/src/parse/_helpers/tree/nodes.rs +++ b/crates/sqlquack-syntax-clickhouse/src/parse/_helpers/tree/nodes.rs @@ -57,7 +57,7 @@ pub(crate) fn star(table: Option, except: Option>) - replace: None, rename: None, trailing_comments: Vec::new(), - span: None, + span: SourceSpan::default(), qualified: None, } } diff --git a/crates/sqlquack-syntax-clickhouse/src/parse/_helpers/tree/spans.rs b/crates/sqlquack-syntax-clickhouse/src/parse/_helpers/tree/spans.rs index b42095824..5012a265f 100644 --- a/crates/sqlquack-syntax-clickhouse/src/parse/_helpers/tree/spans.rs +++ b/crates/sqlquack-syntax-clickhouse/src/parse/_helpers/tree/spans.rs @@ -10,7 +10,7 @@ fn span_field(expression: &mut Expression) -> Option<&mut Option> { match expression { Expression::Column(node) => Some(&mut node.span), Expression::Identifier(node) => Some(&mut node.span), - Expression::Star(node) => Some(&mut node.span), + Expression::Star(node) => Some(&mut node.span.0), Expression::Count(node) => Some(&mut node.span), Expression::Function(node) => Some(&mut node.span.0), Expression::AggregateFunction(node) => Some(&mut node.span.0), diff --git a/crates/sqlquack-syntax-databricks/src/parse/_helpers/tree/nodes.rs b/crates/sqlquack-syntax-databricks/src/parse/_helpers/tree/nodes.rs index 74c8fed30..0f8ea4188 100644 --- a/crates/sqlquack-syntax-databricks/src/parse/_helpers/tree/nodes.rs +++ b/crates/sqlquack-syntax-databricks/src/parse/_helpers/tree/nodes.rs @@ -60,7 +60,7 @@ pub(crate) fn star(table: Option, except: Option>) - replace: None, rename: None, trailing_comments: Vec::new(), - span: None, + span: SourceSpan::default(), qualified: None, } } diff --git a/crates/sqlquack-syntax-databricks/src/parse/_helpers/tree/spans.rs b/crates/sqlquack-syntax-databricks/src/parse/_helpers/tree/spans.rs index b42095824..5012a265f 100644 --- a/crates/sqlquack-syntax-databricks/src/parse/_helpers/tree/spans.rs +++ b/crates/sqlquack-syntax-databricks/src/parse/_helpers/tree/spans.rs @@ -10,7 +10,7 @@ fn span_field(expression: &mut Expression) -> Option<&mut Option> { match expression { Expression::Column(node) => Some(&mut node.span), Expression::Identifier(node) => Some(&mut node.span), - Expression::Star(node) => Some(&mut node.span), + Expression::Star(node) => Some(&mut node.span.0), Expression::Count(node) => Some(&mut node.span), Expression::Function(node) => Some(&mut node.span.0), Expression::AggregateFunction(node) => Some(&mut node.span.0), diff --git a/crates/sqlquack-syntax-duckdb/src/parse/_helpers/tree/nodes.rs b/crates/sqlquack-syntax-duckdb/src/parse/_helpers/tree/nodes.rs index b9b62a851..df34102f3 100644 --- a/crates/sqlquack-syntax-duckdb/src/parse/_helpers/tree/nodes.rs +++ b/crates/sqlquack-syntax-duckdb/src/parse/_helpers/tree/nodes.rs @@ -59,7 +59,7 @@ pub(crate) fn star(table: Option, except: Option>) - replace: None, rename: None, trailing_comments: Vec::new(), - span: None, + span: SourceSpan::default(), qualified: None, } } diff --git a/crates/sqlquack-syntax-duckdb/src/parse/_helpers/tree/spans.rs b/crates/sqlquack-syntax-duckdb/src/parse/_helpers/tree/spans.rs index b42095824..5012a265f 100644 --- a/crates/sqlquack-syntax-duckdb/src/parse/_helpers/tree/spans.rs +++ b/crates/sqlquack-syntax-duckdb/src/parse/_helpers/tree/spans.rs @@ -10,7 +10,7 @@ fn span_field(expression: &mut Expression) -> Option<&mut Option> { match expression { Expression::Column(node) => Some(&mut node.span), Expression::Identifier(node) => Some(&mut node.span), - Expression::Star(node) => Some(&mut node.span), + Expression::Star(node) => Some(&mut node.span.0), Expression::Count(node) => Some(&mut node.span), Expression::Function(node) => Some(&mut node.span.0), Expression::AggregateFunction(node) => Some(&mut node.span.0), diff --git a/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/nodes.rs b/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/nodes.rs index f6ec4a13f..e8e4762b1 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/nodes.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/nodes.rs @@ -39,7 +39,7 @@ pub(crate) fn star(table: Option) -> Expression { replace: None, rename: None, trailing_comments: Vec::new(), - span: None, + span: SourceSpan::default(), qualified: None, })) } diff --git a/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/spans.rs b/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/spans.rs index b42095824..5012a265f 100644 --- a/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/spans.rs +++ b/crates/sqlquack-syntax-postgres/src/parse/_helpers/tree/spans.rs @@ -10,7 +10,7 @@ fn span_field(expression: &mut Expression) -> Option<&mut Option> { match expression { Expression::Column(node) => Some(&mut node.span), Expression::Identifier(node) => Some(&mut node.span), - Expression::Star(node) => Some(&mut node.span), + Expression::Star(node) => Some(&mut node.span.0), Expression::Count(node) => Some(&mut node.span), Expression::Function(node) => Some(&mut node.span.0), Expression::AggregateFunction(node) => Some(&mut node.span.0), diff --git a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/expressions/grouping.rs b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/expressions/grouping.rs index d09688c5d..dc96564cc 100644 --- a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/expressions/grouping.rs +++ b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/expressions/grouping.rs @@ -891,7 +891,7 @@ impl Parser { replace: None, rename: None, trailing_comments: Vec::new(), - span: None, + span: SourceSpan::default(), qualified: None, }, ))); @@ -1013,7 +1013,7 @@ impl Parser { replace: None, rename: None, trailing_comments: Vec::new(), - span: None, + span: SourceSpan::default(), qualified: None, }, ))); @@ -1029,7 +1029,7 @@ impl Parser { replace: None, rename: None, trailing_comments: Vec::new(), - span: None, + span: SourceSpan::default(), qualified: None, }, ))); diff --git a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/expressions/primary.rs b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/expressions/primary.rs index eadf4ef13..24b124043 100644 --- a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/expressions/primary.rs +++ b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/expressions/primary.rs @@ -507,7 +507,7 @@ impl Parser { replace: None, rename: None, trailing_comments: Vec::new(), - span: None, + span: SourceSpan::default(), qualified: None, }, )) @@ -628,7 +628,7 @@ impl Parser { replace: None, rename: None, trailing_comments: Vec::new(), - span: None, + span: SourceSpan::default(), qualified: None, })), right: pattern, @@ -1147,7 +1147,7 @@ impl Parser { replace: None, rename: None, trailing_comments: Vec::new(), - span: None, + span: SourceSpan::default(), qualified: None, })) } else if let Some(lambda) = self.parse_function_lambda()? { diff --git a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/functions/typed.rs b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/functions/typed.rs index 645d54f2a..e56610976 100644 --- a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/functions/typed.rs +++ b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/functions/typed.rs @@ -137,7 +137,7 @@ impl Parser { replace: None, rename: None, trailing_comments: Vec::new(), - span: None, + span: SourceSpan::default(), qualified: None, }))] } else if self.check(TokenType::RParen) { diff --git a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/queries/select.rs b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/queries/select.rs index 8e09566dc..c5d75ca54 100644 --- a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/queries/select.rs +++ b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/queries/select.rs @@ -344,7 +344,7 @@ impl Parser { replace: None, rename: None, trailing_comments: Vec::new(), - span: None, + span: SourceSpan::default(), qualified: None, }, ))] diff --git a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/support/names.rs b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/support/names.rs index 9aaf57397..980cd5364 100644 --- a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/support/names.rs +++ b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/support/names.rs @@ -1305,7 +1305,7 @@ impl Parser { replace, rename, trailing_comments: Vec::new(), - span: None, + span: SourceSpan::default(), qualified: None, })))) } diff --git a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/types/data_types.rs b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/types/data_types.rs index c25fda295..5f54a54d4 100644 --- a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/types/data_types.rs +++ b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/types/data_types.rs @@ -1096,7 +1096,7 @@ impl Parser { replace, rename, trailing_comments: star_trailing_comments, - span: None, + span: SourceSpan::default(), qualified: None, }) } diff --git a/crates/sqlquack-syntax-sqlite/src/parse/_helpers/tree/nodes.rs b/crates/sqlquack-syntax-sqlite/src/parse/_helpers/tree/nodes.rs index 1b25ac2a4..2f91e3e3a 100644 --- a/crates/sqlquack-syntax-sqlite/src/parse/_helpers/tree/nodes.rs +++ b/crates/sqlquack-syntax-sqlite/src/parse/_helpers/tree/nodes.rs @@ -57,7 +57,7 @@ pub(crate) fn star(table: Option, except: Option>) - replace: None, rename: None, trailing_comments: Vec::new(), - span: None, + span: SourceSpan::default(), qualified: None, } } diff --git a/crates/sqlquack-syntax-sqlite/src/parse/_helpers/tree/spans.rs b/crates/sqlquack-syntax-sqlite/src/parse/_helpers/tree/spans.rs index b42095824..5012a265f 100644 --- a/crates/sqlquack-syntax-sqlite/src/parse/_helpers/tree/spans.rs +++ b/crates/sqlquack-syntax-sqlite/src/parse/_helpers/tree/spans.rs @@ -10,7 +10,7 @@ fn span_field(expression: &mut Expression) -> Option<&mut Option> { match expression { Expression::Column(node) => Some(&mut node.span), Expression::Identifier(node) => Some(&mut node.span), - Expression::Star(node) => Some(&mut node.span), + Expression::Star(node) => Some(&mut node.span.0), Expression::Count(node) => Some(&mut node.span), Expression::Function(node) => Some(&mut node.span.0), Expression::AggregateFunction(node) => Some(&mut node.span.0), diff --git a/crates/sqlquack-syntax-sqlserver/src/parse/_helpers/statements/queries.rs b/crates/sqlquack-syntax-sqlserver/src/parse/_helpers/statements/queries.rs index 5d7856f19..f2897b841 100644 --- a/crates/sqlquack-syntax-sqlserver/src/parse/_helpers/statements/queries.rs +++ b/crates/sqlquack-syntax-sqlserver/src/parse/_helpers/statements/queries.rs @@ -64,7 +64,7 @@ fn empty_star(table: Option) -> Star { replace: None, rename: None, trailing_comments: Vec::new(), - span: None, + span: SourceSpan::default(), qualified: None, } } diff --git a/crates/sqlquack-syntax-sqlserver/src/parse/_helpers/values/primary.rs b/crates/sqlquack-syntax-sqlserver/src/parse/_helpers/values/primary.rs index 042d599e4..79aac66c0 100644 --- a/crates/sqlquack-syntax-sqlserver/src/parse/_helpers/values/primary.rs +++ b/crates/sqlquack-syntax-sqlserver/src/parse/_helpers/values/primary.rs @@ -267,7 +267,7 @@ impl<'sql> Parser<'sql> { replace: None, rename: None, trailing_comments: Vec::new(), - span: None, + span: SourceSpan::default(), qualified: None, }))); } else if !self.is_char(')') { diff --git a/crates/sqlquack-syntax/src/expressions/names/models.rs b/crates/sqlquack-syntax/src/expressions/names/models.rs index 35c4e762c..a12c5b80a 100644 --- a/crates/sqlquack-syntax/src/expressions/names/models.rs +++ b/crates/sqlquack-syntax/src/expressions/names/models.rs @@ -11,6 +11,7 @@ use crate::expressions::statements::models::HistoricalData; use crate::expressions::statements::models::Version; use crate::expressions::tree::models::Expression; use crate::expressions::types::models::DataType; +use crate::tokens::models::SourceSpan; use crate::tokens::models::Span; /// A literal, keeping numbers as written so precision, trailing zeros and hex survive. @@ -419,9 +420,10 @@ pub struct Star { /// Trailing comments that appeared after the star #[serde(default, skip_serializing_if = "Vec::is_empty")] pub trailing_comments: Vec, - /// Source position span - #[serde(default, skip_serializing_if = "Option::is_none")] - pub span: Option, + /// The star's source range, recorded only by a located parse. + #[serde(default, skip_serializing_if = "SourceSpan::is_absent")] + #[ast(skip)] + pub span: SourceSpan, /// The name parts of qualified `except` and `rename` entries, with their quoting. #[serde(default, skip_serializing_if = "Option::is_none")] pub qualified: Option>, diff --git a/crates/sqlquack-syntax/src/expressions/tree/_helpers/accessors.rs b/crates/sqlquack-syntax/src/expressions/tree/_helpers/accessors.rs index f7f83be7a..ed6d5fc4b 100644 --- a/crates/sqlquack-syntax/src/expressions/tree/_helpers/accessors.rs +++ b/crates/sqlquack-syntax/src/expressions/tree/_helpers/accessors.rs @@ -12,6 +12,7 @@ use crate::expressions::queries::models::Select; use crate::expressions::tree::main::_type_assertion_result::type_assertion_result; use crate::expressions::tree::models::Expression; use crate::expressions::types::models::DataType; +use crate::tokens::models::SourceSpan; impl Expression { /// Source semantics still needing dialect-aware lowering; kept through serialization. @@ -693,7 +694,7 @@ impl Expression { replace: None, rename: None, trailing_comments: Vec::new(), - span: None, + span: SourceSpan::default(), qualified: None, })) } From df57ec28efdc97d14e92f7d93360f2592d09db32 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 06:03:31 +0000 Subject: [PATCH 2/4] feat(semantics): give each output column its expression's byte range OutputLineage gains expression_range, the UTF-8 byte range in the analysed SQL of the projection item that defines the column, so a caller can highlight it in the SQL it compiled. A column a star brings points at the star, and one a REPLACE gives a value points at that value. Columns merged by a set operation have no single defining item and get none. Spans count Unicode characters, so the ranges are converted to bytes in one pass over the SQL per statement, reading only up to the last offset the outputs need. --- .../src/lineage/_helpers/tracing/queries.rs | 22 +++- .../src/lineage/_helpers/tracing/sources.rs | 6 + .../src/lineage/_helpers/tracing/statement.rs | 54 ++++++++- .../sqlquack-semantics/src/lineage/models.rs | 5 + .../src/lineage/tests/test_types.rs | 10 ++ .../profile/tests/test_lineage_details.rs | 106 +++++++++++++++++- 6 files changed, 198 insertions(+), 5 deletions(-) diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/queries.rs b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/queries.rs index e0490ba41..3583f9fcd 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/queries.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/queries.rs @@ -150,6 +150,7 @@ pub(crate) fn renamed( name: alias.name.clone(), key: key(alias, context), lineage: column.lineage, + expression: column.expression, }, None => column, }) @@ -258,6 +259,7 @@ fn positional(queries: Vec, context: LineageContext<'_>) -> Resol } columns.push(ScopedColumn { lineage: traced_if(set_merged(parts, context.profile), complete), + expression: None, ..column }); } @@ -287,6 +289,7 @@ fn by_name( let column: &mut ScopedColumn = &mut columns[position]; column.lineage = set_merged(vec![column.lineage.clone(), other.lineage], context.profile); + column.expression = None; matched[position] = true; } None => { @@ -543,19 +546,28 @@ fn select_query( let projection = unannotated(projection); if let Expression::Star(star) = projection { let expanded = expanded_star(star, &scope, context); + let star_range = projection.source_span(); query.open |= expanded.open; query.open_relation = query.open_relation.or(expanded.open_relation); query.open_reads = query.open_reads.or(expanded.open_reads); query.outer_nullable |= expanded.outer_nullable; ungrouped.extend(expanded.columns.iter().map(|column| column.lineage.clone())); aggregated.extend(expanded.columns.iter().map(|_| false)); - Rc::make_mut(&mut query.columns).extend(Rc::unwrap_or_clone(expanded.columns)); + Rc::make_mut(&mut query.columns).extend( + Rc::unwrap_or_clone(expanded.columns) + .into_iter() + .map(|column| ScopedColumn { + expression: column.expression.or(star_range), + ..column + }), + ); continue; } let column = ScopedColumn { name: output_name(projection, context), key: output_key(projection, context), lineage: expression_lineage(projection, &scope, context), + expression: projection.source_span(), }; query.rows.rows_of(&column.lineage); if !holds_aggregate(projection) { @@ -860,6 +872,7 @@ fn keys_in_place( name: column.name.clone(), key: column.key.clone(), lineage: merged.lineage.clone(), + expression: None, }); } None => columns.push(column.clone()), @@ -899,6 +912,7 @@ fn star_column( { return ScopedColumn { lineage: expression_lineage(&replacement.this, scope, context), + expression: replacement.this.source_span(), ..column.clone() }; } @@ -907,8 +921,12 @@ fn star_column( name: new.name.clone(), key: key(new, context), lineage: column.lineage.clone(), + expression: None, + }, + None => ScopedColumn { + expression: None, + ..column.clone() }, - None => column.clone(), } } diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs index 8c2798e67..767e51de8 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs @@ -216,6 +216,7 @@ fn joined( name: name.clone(), key: name, lineage, + expression: None, }; match merged_keys .iter_mut() @@ -588,6 +589,7 @@ fn stored_columns( key: context .profile .resolved_identifier(&column.name, column.quoted), + expression: None, lineage: Lineage { data_type: stored_type(column, context), nullability: column.nullability, @@ -641,6 +643,7 @@ fn table_function_source( name: (*column).to_owned(), key: plain_key(column, context), lineage: input.clone(), + expression: None, }) .collect(), ), @@ -701,6 +704,7 @@ pub(crate) fn values_query( key: plain_key(&name, context), name, lineage: values_column(values, position, scope, context), + expression: None, } }) .collect(); @@ -790,6 +794,7 @@ pub(crate) fn pivot_sources( key: context.profile.resolved_identifier(&name, true), name, lineage: aggregated.clone(), + expression: None, }); let query = ResolvedQuery { columns: Rc::new(match dynamic { @@ -890,6 +895,7 @@ fn unpivot_source( name: name.name.clone(), key: key(name, context), lineage, + expression: None, }); let query = ResolvedQuery { columns: Rc::new(passed.into_iter().chain(generated).collect()), diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/statement.rs b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/statement.rs index 475aa20d2..a1b08f077 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/statement.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/statement.rs @@ -1,8 +1,10 @@ //! The column lineage of one parsed statement, within an analysis already running. +use std::ops::Range; use std::rc::Rc; use sqlquack_syntax::Expression; +use sqlquack_syntax::tokens::models::Span; use crate::lineage::_helpers::tracing::queries::{projects_star, resolved_query}; use crate::lineage::models::{LineageContext, OutputLineage, QueryLineage}; @@ -13,8 +15,14 @@ pub(crate) fn analysed_lineage( context: LineageContext<'_>, ) -> QueryLineage { let resolved = resolved_query(statement, &[], None, context); + let columns = Rc::unwrap_or_clone(resolved.columns); + let mut ranges = byte_ranges( + context.sql, + columns.iter().map(|column| column.expression).collect(), + ) + .into_iter(); QueryLineage { - outputs: Rc::unwrap_or_clone(resolved.columns) + outputs: columns .into_iter() .map(|column| OutputLineage { name: column.name, @@ -24,6 +32,7 @@ pub(crate) fn analysed_lineage( resolved: column.lineage.resolved, data_type: column.lineage.data_type.as_deref().map(str::to_owned), nullability: column.lineage.nullability, + expression_range: ranges.next().flatten(), }) .collect(), open_star: resolved.open, @@ -32,3 +41,46 @@ pub(crate) fn analysed_lineage( rows_resolved: !resolved.rows.lost, } } + +/// `spans`, Unicode character ranges in `sql`, as UTF-8 byte ranges, in one pass over `sql`. +fn byte_ranges(sql: Option<&str>, spans: Vec>) -> Vec>> { + let Some(sql) = sql else { + return vec![None; spans.len()]; + }; + let mut offsets: Vec = spans + .iter() + .flatten() + .flat_map(|span| [span.start, span.end]) + .collect(); + offsets.sort_unstable(); + offsets.dedup(); + let mut bytes: Vec<(usize, usize)> = Vec::with_capacity(offsets.len()); + let mut wanted = offsets.iter().copied().peekable(); + for (position, (byte, _)) in sql + .char_indices() + .chain(std::iter::once((sql.len(), ' '))) + .enumerate() + { + while wanted.next_if_eq(&position).is_some() { + bytes.push((position, byte)); + } + if wanted.peek().is_none() { + break; + } + } + let byte = |offset: usize| -> Option { + let index = bytes.partition_point(|&(position, _)| position < offset); + bytes + .get(index) + .filter(|&&(position, _)| position == offset) + .map(|&(_, byte)| byte) + }; + spans + .into_iter() + .map(|span| { + let span = span?; + let (start, end) = (byte(span.start)?, byte(span.end)?); + (start < end).then_some(start..end) + }) + .collect() +} diff --git a/crates/sqlquack-semantics/src/lineage/models.rs b/crates/sqlquack-semantics/src/lineage/models.rs index 2d3d6ed4c..271784f12 100644 --- a/crates/sqlquack-semantics/src/lineage/models.rs +++ b/crates/sqlquack-semantics/src/lineage/models.rs @@ -2,6 +2,7 @@ use std::cell::RefCell; use std::collections::BTreeSet; +use std::ops::Range; use std::rc::Rc; use rustc_hash::FxHashMap; @@ -92,6 +93,8 @@ pub struct OutputLineage { pub data_type: Option, /// Whether it may be NULL. pub nullability: Nullability, + /// The UTF-8 byte range in the analysed SQL of the projection item or star defining it. + pub expression_range: Option>, } /// The lineage of every output column of one query. @@ -146,6 +149,8 @@ pub(crate) struct ScopedColumn { pub(crate) name: String, pub(crate) key: String, pub(crate) lineage: Lineage, + /// The source range of the projection item that defines it, when it has one. + pub(crate) expression: Option, } /// A FROM item: its key, its known columns, and whether it offers others of unknown name. diff --git a/crates/sqlquack-semantics/src/lineage/tests/test_types.rs b/crates/sqlquack-semantics/src/lineage/tests/test_types.rs index 5119fd42f..21b2809f9 100644 --- a/crates/sqlquack-semantics/src/lineage/tests/test_types.rs +++ b/crates/sqlquack-semantics/src/lineage/tests/test_types.rs @@ -28,6 +28,16 @@ pub(crate) struct LineageDetailTestCase { pub(crate) expected_rows_resolved: bool, } +/// A query and, per output, the SQL text its expression range covers, or none. +pub(crate) struct OutputRangeTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) dialect: Dialect, + pub(crate) profile: &'static dyn DialectProfile, + pub(crate) schema: fn() -> ShopSchema, + pub(crate) expected_ranges: &'static [(&'static str, Option<&'static str>)], +} + /// A synthetic shop schema: relation name to its columns. #[derive(Debug)] pub(crate) struct ShopSchema(pub(crate) HashMap>); diff --git a/crates/sqlquack-semantics/src/snowflake/profile/tests/test_lineage_details.rs b/crates/sqlquack-semantics/src/snowflake/profile/tests/test_lineage_details.rs index b7de31148..8e8f77d8b 100644 --- a/crates/sqlquack-semantics/src/snowflake/profile/tests/test_lineage_details.rs +++ b/crates/sqlquack-semantics/src/snowflake/profile/tests/test_lineage_details.rs @@ -1,10 +1,11 @@ use sqlquack_syntax::{Dialect, ParseOptions}; +use crate::dialects::constants::DUCKDB; use crate::dialects::constants::SNOWFLAKE; use crate::lineage::main::query_lineage::query_lineage; use crate::lineage::models::{LineageContext, TransformKind}; -use crate::lineage::tests::helpers::{shop_schema, written}; -use crate::lineage::tests::test_types::LineageDetailTestCase; +use crate::lineage::tests::helpers::{duckdb_shop_schema, shop_schema, written}; +use crate::lineage::tests::test_types::{LineageDetailTestCase, OutputRangeTestCase}; const ALL_ORDER_COLUMNS: [&str; 5] = [ "ORDERS.AMOUNT", @@ -567,3 +568,104 @@ fn given_a_query_when_tracing_lineage_then_values_rows_and_untraced_outputs_are_ ); } } + +#[test] +fn given_a_query_when_tracing_lineage_then_each_output_points_at_the_bytes_that_define_it() { + let test_cases = [ + OutputRangeTestCase { + description: "aliased and bare items after multi-byte text cover their own bytes", + sql: "SELECT 'é—ü' AS label,\n o.amount * 2 AS doubled, o.id FROM orders o", + dialect: Dialect::Snowflake, + profile: &SNOWFLAKE, + schema: shop_schema, + expected_ranges: &[ + ("label", Some("'é—ü' AS label")), + ("doubled", Some("o.amount * 2 AS doubled")), + ("id", Some("o.id")), + ], + }, + OutputRangeTestCase { + description: "columns a star brings from a table or a CTE point at the star", + sql: "WITH c AS (SELECT amount * 2 AS twice, id FROM orders) \ + SELECT 'ü' AS mark, c.* FROM c", + dialect: Dialect::Snowflake, + profile: &SNOWFLAKE, + schema: shop_schema, + expected_ranges: &[ + ("mark", Some("'ü' AS mark")), + ("twice", Some("c.*")), + ("id", Some("c.*")), + ], + }, + OutputRangeTestCase { + description: "a renamed star column points at the star, a replaced one at its value", + sql: "SELECT * EXCLUDE (items, status, customer_id) RENAME (id AS order_id) \ + FROM orders", + dialect: Dialect::Snowflake, + profile: &SNOWFLAKE, + schema: shop_schema, + expected_ranges: &[ + ( + "order_id", + Some("* EXCLUDE (items, status, customer_id) RENAME (id AS order_id)"), + ), + ( + "AMOUNT", + Some("* EXCLUDE (items, status, customer_id) RENAME (id AS order_id)"), + ), + ], + }, + OutputRangeTestCase { + description: "a DuckDB REPLACE value is the replaced column's range", + sql: "SELECT * REPLACE (amount * 2 AS amount) FROM (SELECT id, amount FROM orders)", + dialect: Dialect::DuckDB, + profile: &DUCKDB, + schema: duckdb_shop_schema, + expected_ranges: &[ + ("id", Some("* REPLACE (amount * 2 AS amount)")), + ("amount", Some("amount * 2")), + ], + }, + OutputRangeTestCase { + description: "columns merged by a set operation have no single range", + sql: "SELECT id, 'é' AS kind FROM orders UNION ALL SELECT id, 'c' FROM customers", + dialect: Dialect::Snowflake, + profile: &SNOWFLAKE, + schema: shop_schema, + expected_ranges: &[("id", None), ("kind", None)], + }, + ]; + for test_case in test_cases { + let schema = (test_case.schema)(); + let context = LineageContext { + profile: test_case.profile, + schema: &schema, + sql: Some(test_case.sql), + }; + let lineage = query_lineage( + test_case.sql, + test_case.dialect, + &ParseOptions::default(), + context, + ) + .unwrap_or_else(|error| panic!("{}: {error}", test_case.description)); + let ranges: Vec<(&str, Option<&str>)> = lineage + .outputs + .iter() + .map(|output| { + ( + output.name.as_str(), + output + .expression_range + .clone() + .map(|range| &test_case.sql[range]), + ) + }) + .collect(); + assert_eq!( + ranges, test_case.expected_ranges, + "{}", + test_case.description + ); + } +} From b23c0f7e3c34a1a539342271c085d08486d8ef37 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 06:19:01 +0000 Subject: [PATCH 3/4] fix(semantics): leave every set operation's outputs without a range EXCEPT and MINUS outputs kept the left branch's projection range, and a BY NAME column found in only one branch kept that branch's. None of them has a single defining item, as UNION and INTERSECT outputs already do not, so a set operation now clears the range of every output it returns. Columns a PIVOT or UNPIVOT passes through, or groups by, no longer carry the range of the item that defined them in their input. --- .../src/lineage/_helpers/tracing/queries.rs | 20 +++++++++++++++++-- .../src/lineage/_helpers/tracing/sources.rs | 10 ++++++++-- .../profile/tests/test_lineage_details.rs | 16 +++++++++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/queries.rs b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/queries.rs index 3583f9fcd..88a6db529 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/queries.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/queries.rs @@ -194,7 +194,24 @@ fn set_operation( SetSides::ByName => by_name(left, right, context), SetSides::Intersect => positional(vec![left, right], context), }; - ResolvedQuery { rows, ..query } + ResolvedQuery { + rows, + columns: without_expressions(query.columns), + ..query + } +} + +/// `columns` without the projection items that defined them, as a set operation's outputs. +fn without_expressions(columns: Rc>) -> Rc> { + Rc::new( + Rc::unwrap_or_clone(columns) + .into_iter() + .map(|column| ScopedColumn { + expression: None, + ..column + }) + .collect(), + ) } /// EXCEPT's columns: values and rows from the left, each typed as both branches combine. @@ -289,7 +306,6 @@ fn by_name( let column: &mut ScopedColumn = &mut columns[position]; column.lineage = set_merged(vec![column.lineage.clone(), other.lineage], context.profile); - column.expression = None; matched[position] = true; } None => { diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs index 767e51de8..9cb2f67e6 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs @@ -831,7 +831,10 @@ fn grouped_columns( _ => None, }; match found { - Some(column) => columns.push(column.clone()), + Some(column) => columns.push(ScopedColumn { + expression: None, + ..column.clone() + }), None => known = false, } } @@ -951,7 +954,10 @@ fn passed_columns( .flat_map(|source| source.query.columns.iter()) { if !listed(consumed, &column.key, context) { - passed.push(column.clone()); + passed.push(ScopedColumn { + expression: None, + ..column.clone() + }); } } passed diff --git a/crates/sqlquack-semantics/src/snowflake/profile/tests/test_lineage_details.rs b/crates/sqlquack-semantics/src/snowflake/profile/tests/test_lineage_details.rs index 8e8f77d8b..0e8bbc663 100644 --- a/crates/sqlquack-semantics/src/snowflake/profile/tests/test_lineage_details.rs +++ b/crates/sqlquack-semantics/src/snowflake/profile/tests/test_lineage_details.rs @@ -634,6 +634,22 @@ fn given_a_query_when_tracing_lineage_then_each_output_points_at_the_bytes_that_ schema: shop_schema, expected_ranges: &[("id", None), ("kind", None)], }, + OutputRangeTestCase { + description: "EXCEPT outputs have no single range, though their values come from the left", + sql: "SELECT amount AS a FROM orders EXCEPT SELECT amount FROM orders", + dialect: Dialect::Snowflake, + profile: &SNOWFLAKE, + schema: shop_schema, + expected_ranges: &[("a", None)], + }, + OutputRangeTestCase { + description: "BY NAME outputs have no range, including columns only one branch has", + sql: "SELECT id, status FROM orders UNION ALL BY NAME SELECT id, amount FROM orders", + dialect: Dialect::DuckDB, + profile: &DUCKDB, + schema: duckdb_shop_schema, + expected_ranges: &[("id", None), ("status", None), ("amount", None)], + }, ]; for test_case in test_cases { let schema = (test_case.schema)(); From 0a0d2d55e4d55531b913dc38da5b9c414965f18d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 07:54:34 +0000 Subject: [PATCH 4/4] perf(semantics): map output ranges in ASCII SQL without scanning it Character offsets are byte offsets in ASCII text, so the per-character scan now runs only for SQL holding multi-byte characters. This keeps the analysis instruction count within its recorded baseline. --- .../src/lineage/_helpers/tracing/statement.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/statement.rs b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/statement.rs index a1b08f077..6a29b19c2 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/statement.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/statement.rs @@ -47,6 +47,15 @@ fn byte_ranges(sql: Option<&str>, spans: Vec>) -> Vec = spans .iter() .flatten()