diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/bindings.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/bindings.rs new file mode 100644 index 0000000000..f958c60da0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/bindings.rs @@ -0,0 +1,37 @@ +//! Python's `update_binding_models`: model binding facts among project diagnostics. + +use std::collections::{HashMap, HashSet}; + +/// One diagnostic as `update_binding_models` reads it: code, whether it is a model's, and name. +pub(crate) struct BindingOwner<'a> { + pub(crate) code: &'a str, + pub(crate) is_model: bool, + pub(crate) resource_name: Option<&'a str>, +} + +/// Positions in `diagnostics` of each model's diagnostics whose codes it already binds. +pub(crate) fn binding_positions( + models: &[(&str, HashSet<&str>)], + diagnostics: &[BindingOwner<'_>], +) -> Vec> { + let mut by_model: HashMap<&str, Vec> = HashMap::new(); + for (position, diagnostic) in diagnostics.iter().enumerate() { + if diagnostic.is_model { + by_model + .entry(diagnostic.resource_name.unwrap_or("")) + .or_default() + .push(position); + } + } + let mut result: Vec> = Vec::with_capacity(models.len()); + for (name, codes) in models { + let mut kept: Vec = Vec::new(); + for position in by_model.get(name).map_or(&[][..], Vec::as_slice) { + if codes.contains(diagnostics[*position].code) { + kept.push(*position); + } + } + result.push(kept); + } + result +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/columns.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/columns.rs new file mode 100644 index 0000000000..54fe8f9b87 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/columns.rs @@ -0,0 +1,87 @@ +//! Python's column suggestions: edit distance, evidence order and the closest column. + +use std::cmp::Ordering; + +use crate::semantic_checks::_helpers::text::{ascii, casefold}; +use crate::semantic_checks::constants::{MAX_EDIT_DISTANCE, MIN_ABBREVIATION_LENGTH}; +use crate::semantic_checks::models::SemanticDeferral; + +/// Python's `column_distance`: Levenshtein distance between case-folded names. +pub(crate) fn column_distance(left: &str, right: &str) -> usize { + let left: Vec = casefold(left).into_bytes(); + let right: Vec = casefold(right).into_bytes(); + let mut row: Vec = (0..=right.len()).collect(); + for (i, a) in left.iter().enumerate() { + let mut following: Vec = Vec::with_capacity(right.len() + 1); + following.push(i + 1); + for (j, b) in right.iter().enumerate() { + let last = following[j]; + following.push( + (last + 1) + .min(row[j + 1] + 1) + .min(row[j] + usize::from(a != b)), + ); + } + row = following; + } + row[right.len()] +} + +/// Python's `ordered_columns`: columns by relative edit distance, then by name. +pub(crate) fn ordered_columns<'a>( + name: &str, + columns: &'a [(String, String)], +) -> Result, SemanticDeferral> { + ascii(name)?; + let mut keyed: Vec<(f64, &'a str)> = Vec::with_capacity(columns.len()); + for (column, _) in columns { + ascii(column)?; + let longest = name.len().max(column.len()).max(1); + keyed.push(( + column_distance(name, column) as f64 / longest as f64, + column.as_str(), + )); + } + keyed.sort_by(|left, right| { + left.0 + .partial_cmp(&right.0) + .unwrap_or(Ordering::Equal) + .then_with(|| left.1.cmp(right.1)) + }); + Ok(keyed.into_iter().map(|(_, column)| column).collect()) +} + +/// Python's `closest_column`: the first ordered column within two edits or abbreviating the name. +pub(crate) fn closest_column<'a>( + name: &str, + columns: &'a [(String, String)], +) -> Result, SemanticDeferral> { + let folded_name: String = casefold(name); + for candidate in ordered_columns(name, columns)? { + let distance = column_distance(name, candidate); + let folded_candidate: String = casefold(candidate); + let abbreviation = name.len() >= MIN_ABBREVIATION_LENGTH + && folded_candidate.as_bytes().first() == folded_name.as_bytes().first() + && folded_candidate.as_bytes().last() == folded_name.as_bytes().last() + && is_subsequence(folded_name.as_bytes(), folded_candidate.as_bytes()); + if distance <= MAX_EDIT_DISTANCE || abbreviation { + return Ok(Some(candidate)); + } + } + Ok(None) +} + +/// Python's `all(character in remaining for character in name)` over one shared iterator. +fn is_subsequence(name: &[u8], candidate: &[u8]) -> bool { + let mut position: usize = 0; + for character in name { + match candidate[position..] + .iter() + .position(|value| value == character) + { + Some(offset) => position += offset + 1, + None => return false, + } + } + true +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/completion.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/completion.rs new file mode 100644 index 0000000000..a19d501859 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/completion.rs @@ -0,0 +1,424 @@ +//! Python's `recover_diagnostics`, `explain_diagnostics` and opt-out rejection, in that order. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::LazyLock; + +use regex::Regex; + +use crate::semantic_checks::_helpers::bindings::{BindingOwner, binding_positions}; +use crate::semantic_checks::_helpers::columns::closest_column; +use crate::semantic_checks::_helpers::explain::{ + Binary, ExplainContext, ModelEvidence, Shapes, binary_index, explain_model, is_explained_code, + is_type_code, +}; +use crate::semantic_checks::_helpers::messages::{ + compiled, missing_column, pattern, semantic_help, sentence_message, +}; +use crate::semantic_checks::_helpers::opt_outs::opt_out_diagnostic; +use crate::semantic_checks::_helpers::parsed_sql::{ParsedModelFacts, parsed_model_facts}; +use crate::semantic_checks::_helpers::poison::Poison; +use crate::semantic_checks::_helpers::text::{LineIndex, ascii, plain_text}; +use crate::semantic_checks::constants::{ + MODEL_RESOURCE_TYPE, SEMANTIC_CODE_PREFIX, SQL_TEST_COLUMN_CODE, SQL_TEST_COLUMN_PATTERN, + UNKNOWN_COLUMN_CODE, +}; +use crate::semantic_checks::models::{ + CompletedDiagnostic, CompletionModel, CompletionOutcome, CompletionRequest, FinalDiagnostic, + LineageOutput, SemanticDeferral, SemanticDiagnostic, +}; + +static SQL_TEST_COLUMN: LazyLock> = + LazyLock::new(|| compiled(SQL_TEST_COLUMN_PATTERN)); + +/// Complete recovery, explanation and opt-out rejection, or defer all three to Python. +pub(crate) fn complete(request: &CompletionRequest) -> CompletionOutcome { + match completed(request) { + Ok(outcome) => outcome, + Err(deferral) => CompletionOutcome::Deferred(deferral), + } +} + +/// The request with its shapes and models indexed, and the parsed facts of failing models. +struct Completion<'a> { + request: &'a CompletionRequest, + shapes: Shapes<'a>, + models: HashMap<&'a str, usize>, + parsed: HashMap, +} + +impl<'a> Completion<'a> { + fn new(request: &'a CompletionRequest) -> Result { + let models: HashMap<&'a str, usize> = request + .models + .iter() + .enumerate() + .map(|(index, model)| (model.name.as_str(), index)) + .collect(); + let mut parsed: HashMap = HashMap::new(); + for diagnostic in &request.diagnostics { + let name: &str = diagnostic.resource_name.as_deref().unwrap_or(""); + let Some(index) = models.get(name).copied() else { + continue; + }; + if is_explained_code(&diagnostic.code) && !parsed.contains_key(&index) { + let model = &request.models[index]; + let facts = parsed_model_facts(&model.query_sql, request.dialect.as_deref())?; + parsed.insert(index, facts); + } + } + Ok(Self { + request, + shapes: request + .shapes + .iter() + .map(|(name, columns)| (name.as_str(), columns.as_slice())) + .collect(), + models, + parsed, + }) + } + + fn source(&self, diagnostic: &CompletedDiagnostic) -> &'a SemanticDiagnostic { + &self.request.diagnostics[diagnostic.source] + } + + fn model(&self, diagnostic: &CompletedDiagnostic) -> Option<(usize, &'a CompletionModel)> { + let name: &str = self + .source(diagnostic) + .resource_name + .as_deref() + .unwrap_or(""); + self.models + .get(name) + .map(|index| (*index, &self.request.models[*index])) + } + + fn facts(&self, index: usize) -> Result<&ParsedModelFacts, SemanticDeferral> { + self.parsed + .get(&index) + .ok_or(SemanticDeferral::NativeFailure) + } +} + +fn completed(request: &CompletionRequest) -> Result { + let completion = Completion::new(request)?; + let initial: Vec = request + .diagnostics + .iter() + .enumerate() + .map(|(source, diagnostic)| CompletedDiagnostic { + source, + message: diagnostic.message.clone(), + help: diagnostic.help.clone(), + notes: diagnostic.notes.clone(), + location: diagnostic.location, + line: diagnostic.line, + column: diagnostic.column, + changed: false, + }) + .collect(); + let mut codes: Vec> = request + .models + .iter() + .map(|model| model.binding_codes.iter().map(String::as_str).collect()) + .collect(); + let mut recovered_models: Option>> = None; + let recovered: Vec = match recover(&completion, initial.clone())? { + Some(recovered) => { + let positions = positions(&completion, &recovered, &codes); + codes = model_binding_codes(&completion, &recovered, &positions); + recovered_models = Some(positions); + recovered + } + None => initial, + }; + let (diagnostics, model_bindings) = if recovered.is_empty() { + (recovered, recovered_models) + } else { + let explained = explain(&completion, recovered)?; + let bindings = positions(&completion, &explained, &codes); + (explained, Some(bindings)) + }; + let order = reject_opt_outs(&completion, &diagnostics)?; + Ok(CompletionOutcome::Completed { + diagnostics, + model_bindings, + order, + }) +} + +fn positions( + completion: &Completion<'_>, + diagnostics: &[CompletedDiagnostic], + codes: &[HashSet<&str>], +) -> Vec> { + let mut owners: Vec> = Vec::with_capacity(diagnostics.len()); + for diagnostic in diagnostics { + let source = completion.source(diagnostic); + owners.push(BindingOwner { + code: &source.code, + is_model: source.resource_type.as_deref() == Some(MODEL_RESOURCE_TYPE), + resource_name: source.resource_name.as_deref(), + }); + } + let mut models: Vec<(&str, HashSet<&str>)> = Vec::with_capacity(codes.len()); + for (model, model_codes) in completion.request.models.iter().zip(codes) { + models.push((model.name.as_str(), model_codes.clone())); + } + binding_positions(&models, &owners) +} + +/// The binding codes each model keeps after `update_binding_models`. +fn model_binding_codes<'a>( + completion: &Completion<'a>, + diagnostics: &[CompletedDiagnostic], + positions: &[Vec], +) -> Vec> { + let mut codes: Vec> = Vec::with_capacity(positions.len()); + for kept in positions { + let mut model_codes: HashSet<&'a str> = HashSet::new(); + for position in kept { + model_codes.insert(completion.source(&diagnostics[*position]).code.as_str()); + } + codes.push(model_codes); + } + codes +} + +/// True when `lineage` derives output `column` from `table.column`. +fn reads_input_column(lineage: &[LineageOutput], table: &str, column: &str) -> bool { + for output in lineage { + if output.output_column != column { + continue; + } + for (resource, source) in &output.upstream { + if resource == table && source == column { + return true; + } + } + } + false +} + +/// The poisoned output a B002 root error explains, as `(model, suggestion)`. +fn recovered_output( + completion: &Completion<'_>, + diagnostic: &CompletedDiagnostic, +) -> Result, SemanticDeferral> { + let Some((index, model)) = completion.model(diagnostic) else { + return Ok(None); + }; + let Some((column, table)) = missing_column(&diagnostic.message)? else { + return Ok(None); + }; + if !model.inferred_columns.contains(&column) + || !completion.facts(index)?.unaliased_outputs.contains(&column) + { + return Ok(None); + } + let Some(table) = table else { + return Ok(None); + }; + if !reads_input_column(&model.lineage, &table, &column) { + return Ok(None); + } + let columns: &[(String, String)] = completion + .shapes + .get(table.as_str()) + .copied() + .unwrap_or_default(); + Ok(closest_column(&column, columns)? + .map(|suggestion| (model.name.clone(), suggestion.to_owned()))) +} + +/// Python's `recover_diagnostics`, or `None` when it returns the project unchanged. +fn recover( + completion: &Completion<'_>, + diagnostics: Vec, +) -> Result>, SemanticDeferral> { + let mut poisoned = Poison::default(); + let mut origins: HashMap = HashMap::new(); + let mut has_roots = false; + for diagnostic in &diagnostics { + if completion.source(diagnostic).code != UNKNOWN_COLUMN_CODE { + continue; + } + has_roots = true; + if let Some((model, suggestion)) = recovered_output(completion, diagnostic)? { + let id = completion.source(diagnostic).id; + poisoned.insert(&model, &suggestion, id); + origins.insert(id, (model, suggestion)); + } + } + if !has_roots { + return Ok(None); + } + poisoned.spread( + completion + .request + .models + .iter() + .map(|model| (model.name.as_str(), model.lineage.as_slice())), + ); + let mut skipped: Vec<((usize, String, String), usize)> = Vec::new(); + let mut kept: Vec = Vec::new(); + for diagnostic in diagnostics { + let source = completion.source(&diagnostic); + let target: Option<(String, String)> = recovery_target(&source.code, &diagnostic.message)?; + let root = target + .as_ref() + .and_then(|(table, column)| poisoned.get(table, column)); + match root { + Some(root) if root != source.id => { + let (model, column) = origins.get(&root).cloned().unwrap_or_default(); + let key = (root, model, column); + match skipped.iter().position(|(existing, _)| *existing == key) { + Some(position) => skipped[position].1 += 1, + None => skipped.push((key, 1)), + } + } + _ => kept.push(diagnostic), + } + } + let mut notes: HashMap> = HashMap::new(); + for ((root, model, column), count) in skipped { + notes.entry(root).or_default().push(format!( + "{count} downstream uses of {model}.{column} were not checked because of this error" + )); + } + let mut annotated: Vec = Vec::with_capacity(kept.len()); + for mut diagnostic in kept { + if let Some(extra) = notes.get(&completion.source(&diagnostic).id) { + diagnostic.notes.extend(extra.iter().cloned()); + diagnostic.changed = true; + } + annotated.push(diagnostic); + } + Ok(Some(annotated)) +} + +/// The `(table, column)` use a diagnostic reports, which a poisoned output may explain. +fn recovery_target( + code: &str, + message: &str, +) -> Result, SemanticDeferral> { + let mut target: Option<(String, String)> = None; + if code == UNKNOWN_COLUMN_CODE + && let Some((column, Some(table))) = missing_column(message)? + { + target = Some((table, column)); + } + if code == SQL_TEST_COLUMN_CODE + && let Some(captures) = pattern(&SQL_TEST_COLUMN)?.captures(ascii(message)?) + { + target = Some((captures[1].to_owned(), captures[2].to_owned())); + } + Ok(target) +} + +/// Python's `explain_diagnostics` over a non-empty diagnostic list. +fn explain( + completion: &Completion<'_>, + diagnostics: Vec, +) -> Result, SemanticDeferral> { + let mut binaries: HashMap> = HashMap::new(); + let mut explained: Vec = Vec::with_capacity(diagnostics.len()); + for diagnostic in diagnostics { + let code: &str = &completion.source(&diagnostic).code; + match completion.model(&diagnostic) { + Some((index, model)) if is_explained_code(code) => { + let authored_sql: &str = plain_text(&model.authored_sql)?; + if is_type_code(code) && !binaries.contains_key(&index) { + binaries.insert(index, binary_index(authored_sql)?); + } + let lines = LineIndex::new(authored_sql); + let evidence = ModelEvidence { + authored_sql, + aliases: &completion.facts(index)?.aliases, + lines: &lines, + binaries: binaries.get(&index).map_or(&[], Vec::as_slice), + }; + explained.push(explain_model( + &diagnostic, + &ExplainContext { + code, + model: &evidence, + shapes: &completion.shapes, + dialect: completion.request.dialect.as_deref(), + }, + )?); + } + _ => explained.push(with_semantic_help(diagnostic, code)?), + } + } + Ok(explained) +} + +/// Python's sentence message and code-owned help for a diagnostic outside a model. +fn with_semantic_help( + diagnostic: CompletedDiagnostic, + code: &str, +) -> Result { + let Some(help) = semantic_help(code) else { + return Ok(diagnostic); + }; + let message = sentence_message(&diagnostic.message)?; + let help = match diagnostic.help { + Some(existing) if !existing.is_empty() => existing, + _ => help.to_owned(), + }; + Ok(CompletedDiagnostic { + message, + help: Some(help), + changed: true, + ..diagnostic + }) +} + +/// True when opt-out rejection hides `diagnostic` behind the P009 error of its model. +fn hidden_by_opt_out(source: &SemanticDiagnostic, rejected: &BTreeMap<&str, usize>) -> bool { + let rejected_model = source + .resource_name + .as_deref() + .is_some_and(|name| rejected.contains_key(name)); + let model_resource = source + .resource_type + .as_deref() + .is_none_or(|kind| kind == MODEL_RESOURCE_TYPE); + rejected_model && model_resource && source.code.starts_with(SEMANTIC_CODE_PREFIX) +} + +/// Python's `reject_unneeded_sql_analysis_opt_outs`: the final diagnostic order. +fn reject_opt_outs( + completion: &Completion<'_>, + diagnostics: &[CompletedDiagnostic], +) -> Result, SemanticDeferral> { + let mut rejected: BTreeMap<&str, usize> = BTreeMap::new(); + for (index, model) in completion.request.models.iter().enumerate() { + if model.rejected_opt_out_file.is_some() { + rejected.insert(model.name.as_str(), index); + } + } + let mut order: Vec = Vec::new(); + let mut hidden: HashMap<&str, Vec<&str>> = HashMap::new(); + for (position, diagnostic) in diagnostics.iter().enumerate() { + let source = completion.source(diagnostic); + match source.resource_name.as_deref() { + Some(name) if hidden_by_opt_out(source, &rejected) => { + hidden.entry(name).or_default().push(&source.code); + } + _ => order.push(FinalDiagnostic::Completed(position)), + } + } + for (name, index) in rejected { + let model = &completion.request.models[index]; + let codes: &[&str] = hidden.get(name).map_or(&[], Vec::as_slice); + order.push(FinalDiagnostic::OptOut(opt_out_diagnostic( + index, + &model.name, + model.rejected_opt_out_file.as_deref(), + codes, + )?)); + } + Ok(order) +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explain.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explain.rs new file mode 100644 index 0000000000..8cf8c9f251 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/explain.rs @@ -0,0 +1,305 @@ +//! Python's `_explain_model`: schema evidence, suggestions and operand types for model errors. + +use std::collections::{HashMap, HashSet}; +use std::sync::LazyLock; + +use regex::Regex; + +use crate::semantic_checks::_helpers::columns::{closest_column, ordered_columns}; +use crate::semantic_checks::_helpers::messages::{ + comparison_help, compiled, missing_column, pattern, semantic_help, sentence_message, type_words, +}; +use crate::semantic_checks::_helpers::text::{ + LineIndex, ascii, last_newline_before, newlines_before, prefix, +}; +use crate::semantic_checks::constants::{ + COMPARISON_CODE, DECIMAL_OPERAND_PATTERN, DISPLAY_LIMIT, INTEGER_OPERAND_PATTERN, + OPERAND_PATTERN, QUALIFIER_PATTERN, TEMPORAL_OPERAND_PATTERN, TYPE_CODES, UNKNOWN_COLUMN_CODE, +}; +use crate::semantic_checks::models::{CompletedDiagnostic, SemanticDeferral, SemanticLocation}; + +static QUALIFIER: LazyLock> = LazyLock::new(|| compiled(QUALIFIER_PATTERN)); +static BINARY: LazyLock> = LazyLock::new(|| { + compiled(&format!( + r"(?i)(?P{OPERAND_PATTERN})\s*(?:>=|<=|<>|!=|=|>|<|\+|\*|/|-)\s*(?P{OPERAND_PATTERN})" + )) +}); +static OPERAND_AT: LazyLock> = + LazyLock::new(|| compiled(&format!(r"(?i)\A{OPERAND_PATTERN}"))); +static TEMPORAL_OPERAND: LazyLock> = + LazyLock::new(|| compiled(TEMPORAL_OPERAND_PATTERN)); +static INTEGER_OPERAND: LazyLock> = + LazyLock::new(|| compiled(INTEGER_OPERAND_PATTERN)); +static DECIMAL_OPERAND: LazyLock> = + LazyLock::new(|| compiled(DECIMAL_OPERAND_PATTERN)); + +/// Closed relation shapes by name, each in Python's dict order. +pub(crate) type Shapes<'a> = HashMap<&'a str, &'a [(String, String)]>; + +/// Python's `_TYPE_CODES`: B210 to B219. +pub(crate) fn is_type_code(code: &str) -> bool { + TYPE_CODES.contains(&code) +} + +/// One binary operation Python's `_BINARY` finds in authored SQL. +pub(crate) struct Binary { + start: usize, + end: usize, + left: String, + right: String, +} + +/// Python's `_binary_index`: every binary operation in authored SQL, in order. +pub(crate) fn binary_index(sql: &str) -> Result, SemanticDeferral> { + let binary: &Regex = pattern(&BINARY)?; + let mut found: Vec = Vec::new(); + for captures in binary.captures_iter(sql) { + let (start, end) = captures + .get(0) + .map_or((0, 0), |item| (item.start(), item.end())); + found.push(Binary { + start, + end, + left: captures["left"].to_owned(), + right: captures["right"].to_owned(), + }); + } + Ok(found) +} + +/// What `_explain_model` reads about the model a diagnostic belongs to. +pub(crate) struct ModelEvidence<'a> { + pub(crate) authored_sql: &'a str, + pub(crate) aliases: &'a HashMap, + pub(crate) lines: &'a LineIndex<'a>, + pub(crate) binaries: &'a [Binary], +} + +/// Everything `_explain_model` reads beside the diagnostic itself. +pub(crate) struct ExplainContext<'a> { + pub(crate) code: &'a str, + pub(crate) model: &'a ModelEvidence<'a>, + pub(crate) shapes: &'a Shapes<'a>, + pub(crate) dialect: Option<&'a str>, +} + +/// One model diagnostic rewritten with Python's explanation. +pub(crate) fn explain_model( + diagnostic: &CompletedDiagnostic, + context: &ExplainContext<'_>, +) -> Result { + let ExplainContext { + code, + model, + shapes, + dialect, + } = *context; + let original: String = diagnostic.message.clone(); + let mut message: String = sentence_message(&original)?; + let mut help: Option = semantic_help(code).map(str::to_owned); + if code == COMPARISON_CODE { + help = Some(comparison_help(&type_words(&original)?, dialect)); + } + let mut notes: Vec = diagnostic.notes.clone(); + let mut location: Option = diagnostic.location; + if let Some((name, table)) = missing_column(&original)? { + let columns: &[(String, String)] = shapes + .get(table.as_deref().unwrap_or("")) + .copied() + .unwrap_or_default(); + help = closest_column(&name, columns)? + .filter(|suggestion| !suggestion.is_empty()) + .map(|suggestion| format!("did you mean '{suggestion}'?")); + message = format!("Unknown column '{name}'"); + if let Some(table) = &table { + message.push_str(&format!(" in {table}")); + } + if let Some(location) = &location { + let line: &str = python_line(model.lines, location.line)?; + if let Some(alias) = qualifier(prefix(line, location.column - 1))? + && model.aliases.get(&alias) == table.as_ref() + { + message.push_str(&format!(" (as {alias})")); + } + } + let available: Vec<&str> = ordered_columns(&name, columns)?; + if !available.is_empty() { + let suffix: String = if available.len() > DISPLAY_LIMIT { + format!(", and {} more", available.len() - DISPLAY_LIMIT) + } else { + String::new() + }; + let shown: Vec<&str> = available.iter().take(DISPLAY_LIMIT).copied().collect(); + notes.insert( + 0, + format!( + "{} has: {}{suffix}", + table.as_deref().unwrap_or("None"), + shown.join(", ") + ), + ); + } + } + if is_type_code(code) + && let Some(current) = location + { + let offset: i64 = model.lines.offset(current.line, current.column); + match binary_at(model.binaries, offset) { + Some(binary) => { + let left = operand_type(&binary.left, model.aliases, shapes)?; + let right = operand_type(&binary.right, model.aliases, shapes)?; + if let (Some(left_type), Some(right_type)) = (&left, &right) { + notes.push(format!( + "{} is {left_type}, {} is {right_type}", + binary.left, binary.right + )); + if code == COMPARISON_CODE { + help = Some(comparison_help( + &[left_type.clone(), right_type.clone()], + dialect, + )); + } + } + let mut end: i64 = position(binary.end); + if let (Some(end_line), Some(end_column)) = (current.end_line, current.end_column) { + end = end.max(model.lines.offset(end_line, end_column)); + } + let begin: i64 = position(binary.start).min(offset); + let sql = model.authored_sql; + location = Some(SemanticLocation { + line: newlines_before(sql, begin) + 1, + column: begin - last_newline_before(sql, begin), + end_line: Some(newlines_before(sql, end) + 1), + end_column: Some(end - last_newline_before(sql, end)), + }); + } + None => { + if let Some(operand) = operand_at(model.authored_sql, offset)? + && let Some(kind) = operand_type(operand, model.aliases, shapes)? + { + notes.push(format!("{operand} is {kind}")); + } + } + } + } + Ok(CompletedDiagnostic { + source: diagnostic.source, + message, + help, + notes, + location, + line: location.map_or(diagnostic.line, |current| Some(current.line)), + column: location.map_or(diagnostic.column, |current| Some(current.column)), + changed: true, + }) +} + +/// True when `code` is one `_explain_model` rewrites. +pub(crate) fn is_explained_code(code: &str) -> bool { + code == UNKNOWN_COLUMN_CODE || is_type_code(code) +} + +fn position(offset: usize) -> i64 { + i64::try_from(offset).unwrap_or(i64::MAX) +} + +/// Python's `lines[line - 1]`, including its negative indexing; out of range is not reproduced. +fn python_line<'a>(lines: &'a LineIndex<'a>, line: i64) -> Result<&'a str, SemanticDeferral> { + let count: i64 = i64::try_from(lines.lines.len()).unwrap_or(i64::MAX); + let index: i64 = if line - 1 < 0 { + line - 1 + count + } else { + line - 1 + }; + match usize::try_from(index) { + Ok(index) => lines + .lines + .get(index) + .copied() + .ok_or(SemanticDeferral::UnexpectedPosition), + Err(_) => Err(SemanticDeferral::UnexpectedPosition), + } +} + +/// The alias qualifying the column that ends `prefix`, with doubled quotes undone. +fn qualifier(prefix: &str) -> Result, SemanticDeferral> { + let Some(captures) = pattern(&QUALIFIER)?.captures(prefix) else { + return Ok(None); + }; + let alias: String = match (captures.get(1), captures.get(2)) { + (Some(quoted), _) => quoted.as_str().replace("\"\"", "\""), + (None, Some(plain)) => plain.as_str().to_owned(), + (None, None) => String::new(), + }; + Ok((!alias.is_empty()).then_some(alias)) +} + +/// Python's `_binary_at`: the last operation starting at or before `offset`, if it covers it. +fn binary_at(binaries: &[Binary], offset: i64) -> Option<&Binary> { + let count = binaries.partition_point(|binary| position(binary.start) <= offset); + let binary = binaries.get(count.checked_sub(1)?)?; + (offset < position(binary.end)).then_some(binary) +} + +/// Python's `re.compile(_OPERAND, re.IGNORECASE).match(sql, offset)`. +fn operand_at(sql: &str, offset: i64) -> Result, SemanticDeferral> { + let operand: &Regex = pattern(&OPERAND_AT)?; + let start: usize = usize::try_from(offset.max(0)).unwrap_or(usize::MAX); + Ok(sql + .get(start..) + .and_then(|rest| operand.find(rest)) + .map(|found| found.as_str())) +} + +/// Python's `_operand_type`. +fn operand_type( + text: &str, + aliases: &HashMap, + shapes: &Shapes<'_>, +) -> Result, SemanticDeferral> { + if let Some(captures) = pattern(&TEMPORAL_OPERAND)?.captures(text) { + return Ok(Some(captures[1].to_ascii_uppercase())); + } + if pattern(&INTEGER_OPERAND)?.is_match(text) { + return Ok(Some("INTEGER".to_owned())); + } + if pattern(&DECIMAL_OPERAND)?.is_match(text) { + return Ok(Some("DECIMAL".to_owned())); + } + if text.starts_with('\'') { + return Ok(Some("VARCHAR".to_owned())); + } + if let Some((alias, column)) = text.split_once('.') { + let table: &str = aliases.get(alias).map_or(alias, String::as_str); + let value = shape_type(shapes, table, column); + return match value { + Some(value) if !value.is_empty() => Ok(Some(ascii(value)?.to_ascii_uppercase())), + _ => Ok(None), + }; + } + let tables: HashSet<&str> = aliases.values().map(String::as_str).collect(); + let mut candidates: HashSet = HashSet::new(); + for table in tables { + if let Some(value) = shape_type(shapes, table, text) { + candidates.insert(ascii(value)?.to_ascii_uppercase()); + } + } + Ok(if candidates.len() == 1 { + candidates + .into_iter() + .next() + .filter(|value| !value.is_empty()) + } else { + None + }) +} + +fn shape_type<'a>(shapes: &Shapes<'a>, table: &str, column: &str) -> Option<&'a str> { + let shape: &'a [(String, String)] = shapes.get(table)?; + for (name, value) in shape { + if name == column { + return Some(value.as_str()); + } + } + None +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/messages.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/messages.rs new file mode 100644 index 0000000000..770dd15a7f --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/messages.rs @@ -0,0 +1,114 @@ +//! Python's semantic message rewriting and code-owned help texts. + +use std::sync::LazyLock; + +use regex::{Captures, Regex}; + +use crate::semantic_checks::_helpers::text::ascii; +use crate::semantic_checks::constants::{ + BIGQUERY_DIALECT, COMPARISON_CODE, CONTEXT_SUFFIX_PATTERN, DATE_TYPE, MISSING_PATTERN, + QUOTED_PIECE_PATTERN, SEMANTIC_HELP, TIMESTAMP_TYPE, TYPE_WORDS_PATTERN, +}; +use crate::semantic_checks::models::SemanticDeferral; + +static TYPE_WORDS: LazyLock> = LazyLock::new(|| compiled(TYPE_WORDS_PATTERN)); +static MISSING: LazyLock> = LazyLock::new(|| compiled(MISSING_PATTERN)); +static CONTEXT_SUFFIX: LazyLock> = + LazyLock::new(|| compiled(CONTEXT_SUFFIX_PATTERN)); +static QUOTED_PIECE: LazyLock> = + LazyLock::new(|| compiled(QUOTED_PIECE_PATTERN)); + +/// Compile one of this lane's constant patterns. +pub(crate) fn compiled(pattern: &str) -> Result { + Regex::new(pattern).map_err(|error| error.to_string()) +} + +/// A compiled constant pattern, or a native-failure deferral if it did not compile. +pub(crate) fn pattern( + compiled: &'static Result, +) -> Result<&'static Regex, SemanticDeferral> { + compiled + .as_ref() + .map_err(|_| SemanticDeferral::NativeFailure) +} + +/// A message Python's regexes read the same way, or a deferral. +fn message_text(message: &str) -> Result<&str, SemanticDeferral> { + if message.contains('\n') { + return Err(SemanticDeferral::NonAsciiText); + } + ascii(message) +} + +/// Python's `semantic_help`. +pub(crate) fn semantic_help(code: &str) -> Option<&'static str> { + SEMANTIC_HELP + .iter() + .find(|(known, _)| *known == code) + .map(|(_, help)| *help) +} + +/// Python's `sentence_message`: drop the context suffix and upper-case unquoted type words. +pub(crate) fn sentence_message(message: &str) -> Result { + let suffix: &Regex = pattern(&CONTEXT_SUFFIX)?; + let quoted: &Regex = pattern("ED_PIECE)?; + let words: &Regex = pattern(&TYPE_WORDS)?; + let text = suffix.replace(message_text(message)?, ""); + let mut sentence = String::with_capacity(text.len()); + let mut last: usize = 0; + for piece in quoted.find_iter(&text) { + sentence.push_str(&upper_type_words(words, &text[last..piece.start()])); + sentence.push_str(piece.as_str()); + last = piece.end(); + } + sentence.push_str(&upper_type_words(words, &text[last..])); + Ok(sentence) +} + +fn upper_type_words(words: &Regex, text: &str) -> String { + words.replace_all(text, upper_match).into_owned() +} + +fn upper_match(captures: &Captures<'_>) -> String { + captures[0].to_ascii_uppercase() +} + +/// Python's `re.findall(_TYPE_WORDS, message, re.IGNORECASE)` upper-cased. +pub(crate) fn type_words(message: &str) -> Result, SemanticDeferral> { + let words: &Regex = pattern(&TYPE_WORDS)?; + let mut found: Vec = Vec::new(); + for captures in words.captures_iter(message_text(message)?) { + found.push(captures[1].to_ascii_uppercase()); + } + Ok(found) +} + +/// Python's `missing_column`: the unknown column and its table, if the message names one. +pub(crate) fn missing_column( + message: &str, +) -> Result)>, SemanticDeferral> { + let missing: &Regex = pattern(&MISSING)?; + let Some(captures) = missing.captures(message_text(message)?) else { + return Ok(None); + }; + let table: Option = captures.get(2).map(|table| table.as_str().to_owned()); + Ok(Some((captures[1].to_owned(), table))) +} + +/// Python's `comparison_help`: a concrete literal for a proven temporal comparison mismatch. +pub(crate) fn comparison_help(types: &[String], dialect: Option<&str>) -> String { + if types.iter().any(|value| value.contains(TIMESTAMP_TYPE)) { + let literal = if dialect == Some(BIGQUERY_DIALECT) { + "TIMESTAMP '2026-04-01 00:00:00+00'" + } else { + "TIMESTAMP '2026-04-01'" + }; + return format!("compare with a timestamp, for example {literal}"); + } + if types.iter().any(|value| value == DATE_TYPE) { + return "compare with a date, for example DATE '2026-04-01'".to_owned(); + } + semantic_help(COMPARISON_CODE) + .unwrap_or_default() + .to_owned() +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/mod.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/mod.rs new file mode 100644 index 0000000000..166cab73d1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod bindings; +pub(crate) mod columns; +pub(crate) mod completion; +pub(crate) mod explain; +pub(crate) mod messages; +pub(crate) mod opt_outs; +pub(crate) mod parsed_sql; +pub(crate) mod poison; +pub(crate) mod text; +pub(crate) mod type_recovery; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/opt_outs.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/opt_outs.rs new file mode 100644 index 0000000000..87d73a9db9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/opt_outs.rs @@ -0,0 +1,90 @@ +//! Python's `unneeded_opt_out_diagnostic` text for a rejected model opt-out. + +use std::sync::LazyLock; + +use regex::Regex; + +use crate::semantic_checks::_helpers::messages::{compiled, pattern}; +use crate::semantic_checks::constants::{ + ADDITIONAL_HELP_SEPARATOR, FINDING_KINDS, PROJECT_CONFIG_FILENAME, REQUIRE_SQL_ANALYSIS_KEY, + SETTING_SNIPPET_INDENT, SETTINGS_SECTION, +}; +use crate::semantic_checks::models::{OptOutDiagnostic, SemanticDeferral}; + +static FINDING_PATTERNS: LazyLock> = LazyLock::new(|| { + let alternatives: Vec = FINDING_KINDS + .iter() + .map(|(code, _, _)| format!("({code})")) + .collect(); + compiled(&alternatives.join("|")) +}); + +/// One P009 error for a model's rejected `sql_analysis false`, counting the findings it hid. +pub(crate) fn opt_out_diagnostic( + model: usize, + name: &str, + location_file: Option<&str>, + hidden_codes: &[&str], +) -> Result { + let findings: String = finding_counts(pattern(&FINDING_PATTERNS)?, hidden_codes); + let opt_out: &str = if location_file == Some(PROJECT_CONFIG_FILENAME) { + "`sql_analysis = false` from this [path_defaults] entry" + } else { + "`sql_analysis false`" + }; + let fix: String = if findings.is_empty() { + format!("remove {opt_out}; it is not hiding any findings") + } else { + format!( + "remove {opt_out} and fix the findings it was hiding: {findings} \ + (run `sqb compile` to see them)" + ) + }; + let setting_help: String = format!( + "to allow `sql_analysis false` on any model, SQL test or audit, set this in \ + {PROJECT_CONFIG_FILENAME}:\n{SETTING_SNIPPET_INDENT}[{SETTINGS_SECTION}]\n\ + {SETTING_SNIPPET_INDENT}{REQUIRE_SQL_ANALYSIS_KEY} = false" + ); + Ok(OptOutDiagnostic { + model, + message: format!("`sql_analysis false` is not needed for model '{name}'"), + note: format!( + "{PROJECT_CONFIG_FILENAME} sets [{SETTINGS_SECTION}] {REQUIRE_SQL_ANALYSIS_KEY} = true\ + , which only allows `sql_analysis false` on SQL that SQLBuild cannot parse; \ + this model parses successfully." + ), + help: format!("{fix}{ADDITIONAL_HELP_SEPARATOR}{setting_help}"), + }) +} + +/// Python's `_finding_counts`: kinds by descending count, ties in first-seen order. +fn finding_counts(kinds: &Regex, codes: &[&str]) -> String { + let mut counts: Vec<((&str, &str), usize)> = Vec::new(); + for code in codes { + let kind = finding_kind(kinds, code); + match counts.iter_mut().find(|(existing, _)| *existing == kind) { + Some((_, count)) => *count += 1, + None => counts.push((kind, 1)), + } + } + counts.sort_by_key(|(_, count)| std::cmp::Reverse(*count)); + let mut rendered: Vec = Vec::with_capacity(counts.len()); + for ((singular, plural), count) in counts { + let noun = if count == 1 { singular } else { plural }; + rendered.push(format!("{count} {noun}")); + } + rendered.join(", ") +} + +/// Python's `_finding_kind`: the first pattern matching `code`, as one alternation group. +fn finding_kind(kinds: &Regex, code: &str) -> (&'static str, &'static str) { + let Some(captures) = kinds.captures(code) else { + return ("other finding", "other findings"); + }; + for (index, (_, singular, plural)) in FINDING_KINDS.iter().enumerate() { + if captures.get(index + 1).is_some() { + return (singular, plural); + } + } + ("other finding", "other findings") +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/parsed_sql.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/parsed_sql.rs new file mode 100644 index 0000000000..6fb68b7e1f --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/parsed_sql.rs @@ -0,0 +1,179 @@ +//! The wheel reads semantic completion makes of model SQL: projection spans and parsed facts. + +use std::collections::{HashMap, HashSet}; + +use polyglot_sql::tokens::TokenType; +use polyglot_sql::{ + ComplexityGuardOptions, Dialect, DialectType, Expression, ExpressionWalk, ParseOptions, +}; + +use crate::semantic_checks::constants::{ + CLOSE_PARENTHESIS, GENERIC_DIALECT, MAX_FUNCTION_CALL_DEPTH, OPEN_PARENTHESIS, + PROJECTION_SEPARATOR, +}; +use crate::semantic_checks::models::SemanticDeferral; +use crate::semantic_validation::main::normalize::normalize_analysis_sql; +use crate::semantic_validation::models::NormalizationInput; + +/// The Polyglot dialect the wheel resolves `name` to, when this build includes it. +pub(crate) fn polyglot_dialect(name: &str) -> Result { + let Ok(dialect_type) = name.parse::() else { + return Err(SemanticDeferral::UnsupportedDialect); + }; + if matches!( + dialect_type, + DialectType::Generic + | DialectType::PostgreSQL + | DialectType::BigQuery + | DialectType::Snowflake + | DialectType::DuckDB + | DialectType::TSQL + | DialectType::Databricks + ) { + Ok(Dialect::get(dialect_type)) + } else { + Err(SemanticDeferral::UnsupportedDialect) + } +} + +/// Python's `get_complete_schema_binding_request(...).sql` without placeholders. +pub(crate) fn normalized_sql( + query_sql: &str, + dialect: Option<&str>, +) -> Result { + normalize_analysis_sql(NormalizationInput { + sql: query_sql.to_owned(), + dialect: dialect.unwrap_or(GENERIC_DIALECT).to_owned(), + stubs: HashMap::new(), + placeholders: HashMap::new(), + }) + .map_err(|_| SemanticDeferral::UnreadableSql) +} + +/// Python's `_projection_spans`: top-level SELECT-list item spans in code points. +pub(crate) fn projection_spans( + sql: &str, + dialect: Option<&str>, +) -> Result, SemanticDeferral> { + let Some(name) = dialect else { + return Err(SemanticDeferral::UnsupportedDialect); + }; + let tokens = polyglot_dialect(name)? + .tokenize(sql) + .map_err(|_| SemanticDeferral::UnreadableSql)?; + let mut depth: i64 = 0; + let mut start: Option = None; + let mut spans: Vec<(usize, usize)> = Vec::new(); + for token in tokens { + if token.text == OPEN_PARENTHESIS { + depth += 1; + } else if token.text == CLOSE_PARENTHESIS { + depth -= 1; + } else if depth == 0 { + match start { + None if token.token_type == TokenType::Select => start = Some(token.span.end), + Some(begin) if projection_end(token.token_type) => { + spans.push((begin, token.span.start)); + return Ok(spans); + } + Some(begin) if token.text == PROJECTION_SEPARATOR => { + spans.push((begin, token.span.start)); + start = Some(token.span.end); + } + _ => {} + } + } + } + if let Some(begin) = start { + spans.push((begin, sql.chars().count())); + } + Ok(spans) +} + +fn projection_end(token_type: TokenType) -> bool { + matches!( + token_type, + TokenType::From + | TokenType::Where + | TokenType::GroupBy + | TokenType::Having + | TokenType::Qualify + | TokenType::OrderBy + | TokenType::Limit + | TokenType::Union + | TokenType::Intersect + | TokenType::Except + ) +} + +/// The facts `input_aliases` and `unaliased_output_columns` read from one parsed model. +#[derive(Debug, Default)] +pub(crate) struct ParsedModelFacts { + pub(crate) aliases: HashMap, + pub(crate) unaliased_outputs: HashSet, +} + +/// Python's `_parsed_model` facts; SQL the wheel cannot parse has none. +pub(crate) fn parsed_model_facts( + query_sql: &str, + dialect: Option<&str>, +) -> Result { + let sql = normalized_sql(query_sql, dialect)?; + let parser = polyglot_dialect(dialect.unwrap_or(GENERIC_DIALECT))?; + let guard: ComplexityGuardOptions = serde_json::from_value(serde_json::json!({ + "maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH, + })) + .map_err(|_| SemanticDeferral::NativeFailure)?; + let options = ParseOptions { + complexity_guard: Some(guard), + }; + let Ok(mut statements) = parser.parse_with_options(&sql, &options) else { + return Ok(ParsedModelFacts::default()); + }; + if statements.len() != 1 { + return Ok(ParsedModelFacts::default()); + } + let parsed: Expression = statements.remove(0); + Ok(ParsedModelFacts { + aliases: input_aliases(&parsed), + unaliased_outputs: unaliased_outputs(&parsed), + }) +} + +fn input_aliases(parsed: &Expression) -> HashMap { + let mut aliases: HashMap> = HashMap::new(); + for node in parsed.dfs().skip(1) { + let Expression::Table(table) = node else { + continue; + }; + let name = table.name.name.clone(); + aliases + .entry(name.clone()) + .or_default() + .insert(name.clone()); + if let Some(alias) = &table.alias { + aliases.entry(alias.name.clone()).or_default().insert(name); + } + } + let mut unique: HashMap = HashMap::new(); + for (alias, names) in aliases { + let mut names = names.into_iter(); + if let (Some(name), None) = (names.next(), names.next()) { + unique.insert(alias, name); + } + } + unique +} + +fn unaliased_outputs(parsed: &Expression) -> HashSet { + let Expression::Select(select) = parsed else { + return HashSet::new(); + }; + let mut outputs: HashSet = HashSet::new(); + for expression in &select.expressions { + if let Expression::Column(column) = expression { + outputs.insert(column.name.name.clone()); + } + } + outputs +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/poison.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/poison.rs new file mode 100644 index 0000000000..19493fd125 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/poison.rs @@ -0,0 +1,75 @@ +//! Poisoned model outputs in Python's dict order, spread along compiled lineage. + +use std::collections::HashMap; + +use crate::semantic_checks::models::LineageOutput; + +/// Python's `dict[tuple[str, str], CompilerDiagnostic]` keyed by `(model, column)`. +#[derive(Debug, Default)] +pub(crate) struct Poison { + order: Vec<(String, String)>, + roots: HashMap<(String, String), usize>, +} + +impl Poison { + pub(crate) fn len(&self) -> usize { + self.order.len() + } + + pub(crate) fn is_empty(&self) -> bool { + self.order.is_empty() + } + + pub(crate) fn get(&self, model: &str, column: &str) -> Option { + self.roots + .get(&(model.to_owned(), column.to_owned())) + .copied() + } + + /// `poisoned[key] = root`: a new key goes last, an existing key keeps its place. + pub(crate) fn insert(&mut self, model: &str, column: &str, root: usize) { + let key = (model.to_owned(), column.to_owned()); + if self.roots.insert(key.clone(), root).is_none() { + self.order.push(key); + } + } + + /// `poisoned.setdefault(key, root)`. + pub(crate) fn set_default(&mut self, model: &str, column: &str, root: usize) { + let key = (model.to_owned(), column.to_owned()); + if !self.roots.contains_key(&key) { + self.roots.insert(key.clone(), root); + self.order.push(key); + } + } + + /// Entries in insertion order. + pub(crate) fn entries(&self) -> impl Iterator { + self.order + .iter() + .map(|key| (key, self.roots.get(key).copied().unwrap_or_default())) + } + + /// Python's bounded fixed point: one pass per model, stopping when a pass adds nothing. + pub(crate) fn spread<'a>( + &mut self, + models: impl Iterator + Clone, + ) { + let passes = models.clone().count(); + for _ in 0..passes { + let previous = self.len(); + for (name, lineage) in models.clone() { + for output in lineage { + for (resource, column) in &output.upstream { + if let Some(root) = self.get(resource, column) { + self.set_default(name, &output.output_column, root); + } + } + } + } + if self.len() == previous { + break; + } + } + } +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/text.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/text.rs new file mode 100644 index 0000000000..9537c2ac40 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/text.rs @@ -0,0 +1,114 @@ +//! Python string operations over ASCII text, where byte offsets equal code-point offsets. + +use crate::semantic_checks::models::SemanticDeferral; + +/// ASCII text or a deferral, because Python's case folding and regex classes differ beyond it. +pub(crate) fn ascii(text: &str) -> Result<&str, SemanticDeferral> { + if text.is_ascii() { + Ok(text) + } else { + Err(SemanticDeferral::NonAsciiText) + } +} + +/// ASCII text without the separators Python's `\s` matches and Rust's does not. +pub(crate) fn plain_text(text: &str) -> Result<&str, SemanticDeferral> { + if ascii(text)? + .bytes() + .any(|byte| (0x1c..=0x1f).contains(&byte)) + { + return Err(SemanticDeferral::NonAsciiText); + } + Ok(text) +} + +/// Python's `str.casefold` on ASCII text. +pub(crate) fn casefold(text: &str) -> String { + text.to_ascii_lowercase() +} + +/// Python's `str.splitlines(keepends=True)` line lengths on ASCII text. +fn line_ends(text: &str) -> Vec<(usize, usize)> { + let bytes = text.as_bytes(); + let mut lines: Vec<(usize, usize)> = Vec::new(); + let mut start: usize = 0; + let mut index: usize = 0; + while index < bytes.len() { + let byte = bytes[index]; + if matches!(byte, b'\n' | b'\r' | 0x0b | 0x0c | 0x1c | 0x1d | 0x1e) { + let content_end = index; + index += 1; + if byte == b'\r' && bytes.get(index) == Some(&b'\n') { + index += 1; + } + lines.push((start, content_end)); + start = index; + } else { + index += 1; + } + } + if start < bytes.len() { + lines.push((start, bytes.len())); + } + lines +} + +/// Python's `_line_index`: `splitlines()` and the running starts of `splitlines(keepends=True)`. +pub(crate) struct LineIndex<'a> { + pub(crate) lines: Vec<&'a str>, + pub(crate) starts: Vec, +} + +impl<'a> LineIndex<'a> { + pub(crate) fn new(text: &'a str) -> Self { + let ends = line_ends(text); + let mut starts: Vec = vec![0]; + let mut lines: Vec<&'a str> = Vec::with_capacity(ends.len()); + for (index, (start, content_end)) in ends.iter().enumerate() { + lines.push(&text[*start..*content_end]); + let next_start = ends.get(index + 1).map_or(text.len(), |(next, _)| *next); + let previous = starts.last().copied().unwrap_or(0); + starts.push(previous + (next_start - start)); + } + Self { lines, starts } + } + + /// Python's `_line_offset`, clamping the line into the index. + pub(crate) fn offset(&self, line: i64, column: i64) -> i64 { + let last: i64 = i64::try_from(self.starts.len()).unwrap_or(i64::MAX) - 1; + let position: usize = usize::try_from((line - 1).max(0).min(last)).unwrap_or(0); + i64::try_from(self.starts[position]).unwrap_or(i64::MAX) + column - 1 + } +} + +/// Python's `text.count("\n", 0, end)` with Python's slice clamping. +pub(crate) fn newlines_before(text: &str, end: i64) -> i64 { + let end: usize = clamp(text, end); + i64::try_from( + text.as_bytes()[..end] + .iter() + .filter(|byte| **byte == b'\n') + .count(), + ) + .unwrap_or(i64::MAX) +} + +/// Python's `text.rfind("\n", 0, end)` with Python's slice clamping. +pub(crate) fn last_newline_before(text: &str, end: i64) -> i64 { + let end: usize = clamp(text, end); + text.as_bytes()[..end] + .iter() + .rposition(|byte| *byte == b'\n') + .map_or(-1, |index| i64::try_from(index).unwrap_or(i64::MAX)) +} + +fn clamp(text: &str, end: i64) -> usize { + let length: i64 = i64::try_from(text.len()).unwrap_or(i64::MAX); + let end: i64 = if end < 0 { (length + end).max(0) } else { end }; + usize::try_from(end.min(length)).unwrap_or(0) +} + +/// Python's `text[:end]` for a non-negative end on ASCII text. +pub(crate) fn prefix(text: &str, end: i64) -> &str { + &text[..clamp(text, end)] +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/type_recovery.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/type_recovery.rs new file mode 100644 index 0000000000..9e1446429b --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/type_recovery.rs @@ -0,0 +1,290 @@ +//! Python's `recover_output_types`: unknown types for outputs poisoned by located root errors. + +use std::collections::{HashMap, HashSet, VecDeque}; + +use crate::semantic_checks::_helpers::bindings::{BindingOwner, binding_positions}; +use crate::semantic_checks::_helpers::parsed_sql::{normalized_sql, projection_spans}; +use crate::semantic_checks::_helpers::poison::Poison; +use crate::semantic_checks::models::{ + RecoveryModel, SemanticDeferral, TypeRecoveryOutcome, TypeRecoveryPlan, TypeRecoveryRequest, + TypeRecoveryStep, +}; +use crate::semantic_checks::types::RevisedBinding; + +/// Find poisoned outputs and the failed models Python must revalidate with unknown types. +pub(crate) fn plan(request: &TypeRecoveryRequest) -> TypeRecoveryStep { + match planned(request) { + Ok(step) => step, + Err(deferral) => TypeRecoveryStep::Deferred(deferral), + } +} + +fn planned(request: &TypeRecoveryRequest) -> Result { + let failed: Vec = failed_models(request); + if !has_blocking_error(request, &failed) { + return Ok(TypeRecoveryStep::Unchanged); + } + let dialect = request.dialect.as_deref(); + let sqls: Vec = failed + .iter() + .map(|index| normalized_sql(&request.models[*index].query_sql, dialect)) + .collect::>()?; + let errors: HashMap = error_ids(request); + let mut poisoned = Poison::default(); + let mut roots_by_model: HashMap> = HashMap::new(); + let mut outputs_by_diagnostic: HashMap> = HashMap::new(); + for (index, sql) in failed.iter().zip(&sqls) { + let model: &RecoveryModel = &request.models[*index]; + let roots = root_bindings(model); + let spans = projection_spans(sql, dialect)?; + let columns: &[String] = model.inferred_columns.as_deref().unwrap_or_default(); + if spans.len() == columns.len() { + let starts: Vec = spans.iter().map(|(start, _)| *start).collect(); + for raw in &model.raw_bindings { + let Some(offset) = raw.start else { + continue; + }; + let root = roots.get(&raw.id).copied(); + let column = column_at(columns, &spans, &starts, offset); + if let (Some(root), Some(column)) = (root, column) + && errors.get(&root).copied().unwrap_or(false) + { + poisoned.insert(&model.name, column, root); + outputs_by_diagnostic + .entry(root) + .or_default() + .insert(column.clone()); + } + } + } + roots_by_model.insert(*index, roots); + } + if poisoned.is_empty() { + return Ok(TypeRecoveryStep::Unchanged); + } + poisoned.spread( + request + .models + .iter() + .map(|model| (model.name.as_str(), model.lineage.as_slice())), + ); + let mut causes_by_model: Vec<(usize, HashSet)> = Vec::new(); + for index in &failed { + let causes = reference_causes(&request.models[*index], &poisoned); + if !causes.is_empty() { + causes_by_model.push((*index, causes)); + } + } + Ok(TypeRecoveryStep::Planned(TypeRecoveryPlan { + poisoned: poisoned + .entries() + .map(|(key, root)| (key.clone(), root)) + .collect(), + roots_by_model, + outputs_by_diagnostic, + causes_by_model, + })) +} + +/// Retain diagnostics the revalidation still reports and count unchecked downstream uses. +pub(crate) fn finish( + request: &TypeRecoveryRequest, + plan: &TypeRecoveryPlan, + revised: &[Vec], +) -> TypeRecoveryOutcome { + let mut poisoned = Poison::default(); + for ((model, column), root) in &plan.poisoned { + poisoned.insert(model, column, *root); + } + let mut skipped: HashSet = HashSet::new(); + let mut redirected: HashMap> = HashMap::new(); + let empty: HashMap = HashMap::new(); + for ((index, causes), rows) in plan.causes_by_model.iter().zip(revised) { + let model: &RecoveryModel = &request.models[*index]; + let remaining: HashSet<&RevisedBinding> = rows.iter().collect(); + let roots = plan.roots_by_model.get(index).unwrap_or(&empty); + for raw in &model.raw_bindings { + let Some(diagnostic) = roots.get(&raw.id).copied() else { + continue; + }; + let key: RevisedBinding = (raw.code.clone(), raw.start, raw.end); + if causes.contains(&diagnostic) || remaining.contains(&key) { + continue; + } + skipped.insert(diagnostic); + let outputs = plan.outputs_by_diagnostic.get(&diagnostic); + let mut redirect = output_causes(model, outputs, &poisoned); + if redirect.is_empty() { + redirect = causes.clone(); + } + redirected.insert(diagnostic, redirect); + } + } + let counts = unchecked_uses(request, &poisoned, &redirected); + let mut kept: Vec<(usize, Option)> = Vec::new(); + let mut owners: Vec> = Vec::new(); + for (position, diagnostic) in request.diagnostics.iter().enumerate() { + if skipped.contains(&diagnostic.id) { + continue; + } + let count: usize = counts.get(&diagnostic.id).copied().unwrap_or_default(); + let note: Option = (count > 0).then(|| { + format!("{count} downstream output uses were not type-checked because of this error") + }); + kept.push((position, note)); + owners.push(BindingOwner { + code: &diagnostic.code, + is_model: diagnostic.is_model, + resource_name: diagnostic.resource_name.as_deref(), + }); + } + let mut models: Vec<(&str, HashSet<&str>)> = Vec::with_capacity(request.models.len()); + for model in &request.models { + let codes: HashSet<&str> = model + .bindings + .iter() + .map(|binding| binding.code.as_str()) + .collect(); + models.push((model.name.as_str(), codes)); + } + TypeRecoveryOutcome { + model_bindings: binding_positions(&models, &owners), + kept, + } +} + +fn has_blocking_error(request: &TypeRecoveryRequest, failed: &[usize]) -> bool { + for index in failed { + if request.models[*index] + .bindings + .iter() + .any(|binding| binding.is_error) + { + return true; + } + } + false +} + +fn failed_models(request: &TypeRecoveryRequest) -> Vec { + request + .models + .iter() + .enumerate() + .filter(|(_, model)| !model.bindings.is_empty()) + .map(|(index, _)| index) + .collect() +} + +fn error_ids(request: &TypeRecoveryRequest) -> HashMap { + let mut errors: HashMap = HashMap::new(); + for model in &request.models { + for binding in &model.bindings { + errors.insert(binding.id, binding.is_error); + } + } + errors +} + +/// Python's `_root_bindings`: pair repeated messages by occurrence. +fn root_bindings(model: &RecoveryModel) -> HashMap { + let mut pending: HashMap<(&str, &str), VecDeque> = HashMap::new(); + for binding in &model.bindings { + pending + .entry((binding.code.as_str(), binding.message.as_str())) + .or_default() + .push_back(binding.id); + } + let mut result: HashMap = HashMap::new(); + for raw in &model.raw_bindings { + if let Some(root) = pending + .get_mut(&(raw.code.as_str(), raw.message.as_str())) + .and_then(VecDeque::pop_front) + { + result.insert(raw.id, root); + } + } + result +} + +/// Python's `_column_at`: the only projection span that can contain `offset`. +fn column_at<'a>( + columns: &'a [String], + spans: &[(usize, usize)], + starts: &[usize], + offset: i64, +) -> Option<&'a String> { + let reached: usize = match usize::try_from(offset) { + Ok(offset) => starts.partition_point(|start| *start <= offset), + Err(_) => 0, + }; + let position = reached.checked_sub(1)?; + let end: i64 = i64::try_from(spans[position].1).unwrap_or(i64::MAX); + if offset >= end { + return None; + } + columns.get(position) +} + +fn reference_causes(model: &RecoveryModel, poisoned: &Poison) -> HashSet { + let mut causes: HashSet = HashSet::new(); + for reference in &model.references { + for ((name, _), root) in poisoned.entries() { + if name == reference { + causes.insert(root); + } + } + } + causes +} + +/// Python's `_output_causes`: the poisoned inputs of the failing projection only. +fn output_causes( + model: &RecoveryModel, + outputs: Option<&HashSet>, + poisoned: &Poison, +) -> HashSet { + let mut result: HashSet = HashSet::new(); + let Some(outputs) = outputs else { + return result; + }; + for output in &model.lineage { + if outputs.contains(&output.output_column) { + for (resource, column) in &output.upstream { + if let Some(root) = poisoned.get(resource, column) { + result.insert(root); + } + } + } + } + result +} + +/// Python's `_unchecked_uses`: transitive unchecked output uses per retained root error. +fn unchecked_uses( + request: &TypeRecoveryRequest, + poisoned: &Poison, + redirected: &HashMap>, +) -> HashMap { + let mut counts: HashMap = HashMap::new(); + for model in &request.models { + for output in &model.lineage { + let mut pending: Vec = output + .upstream + .iter() + .filter_map(|(resource, column)| poisoned.get(resource, column)) + .collect(); + let mut visited: HashSet = HashSet::new(); + while let Some(root) = pending.pop() { + if !visited.insert(root) { + continue; + } + match redirected.get(&root) { + Some(targets) => pending.extend(targets.iter().copied()), + None => *counts.entry(root).or_default() += 1, + } + } + } + } + counts +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/constants.rs b/crates/sqlbuild-analysis/src/semantic_checks/constants.rs new file mode 100644 index 0000000000..d42aef5b0c --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/constants.rs @@ -0,0 +1,142 @@ +//! Codes, help texts and patterns Python's semantic completion uses. + +pub(crate) const UNKNOWN_COLUMN_CODE: &str = "B002"; +pub(crate) const COMPARISON_CODE: &str = "B217"; +pub(crate) const SQL_TEST_COLUMN_CODE: &str = "B302"; +pub(crate) const SEMANTIC_CODE_PREFIX: &str = "B"; +pub(crate) const MODEL_RESOURCE_TYPE: &str = "model"; +pub(crate) const MAX_FUNCTION_CALL_DEPTH: u64 = 512; +pub(crate) const GENERIC_DIALECT: &str = "generic"; +pub(crate) const OPEN_PARENTHESIS: &str = "("; +pub(crate) const CLOSE_PARENTHESIS: &str = ")"; +pub(crate) const PROJECTION_SEPARATOR: &str = ","; +pub(crate) const TYPE_CODES: [&str; 10] = [ + "B210", "B211", "B212", "B213", "B214", "B215", "B216", "B217", "B218", "B219", +]; +pub(crate) const MIN_ABBREVIATION_LENGTH: usize = 3; +pub(crate) const MAX_EDIT_DISTANCE: usize = 2; +pub(crate) const DISPLAY_LIMIT: usize = 10; +pub(crate) const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +pub(crate) const SETTINGS_SECTION: &str = "settings"; +pub(crate) const REQUIRE_SQL_ANALYSIS_KEY: &str = "require_sql_analysis"; +pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; +pub(crate) const ADDITIONAL_HELP_SEPARATOR: &str = "\n = help: "; +pub(crate) const TIMESTAMP_TYPE: &str = "TIMESTAMP"; +pub(crate) const DATE_TYPE: &str = "DATE"; +pub(crate) const BIGQUERY_DIALECT: &str = "bigquery"; + +pub(crate) const TYPE_WORDS_PATTERN: &str = r"(?i)\b(timestamp|integer|varchar|boolean|date|interval|double|float|decimal|numeric|bigint|smallint|text|time|string|binary|array|struct)\b"; +pub(crate) const MISSING_PATTERN: &str = r"Unknown column '([^']+)'(?: in table '([^']+)')?"; +pub(crate) const CONTEXT_SUFFIX_PATTERN: &str = r" \(context: [^)]*\)$"; +pub(crate) const QUOTED_PIECE_PATTERN: &str = r"'[^']*'"; +pub(crate) const OPERAND_PATTERN: &str = r"(?:TIMESTAMP\s+'[^']*'|DATE\s+'[^']*'|'(?:[^']|'')*'|-?\d+(?:\.\d+)?|[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)?)"; +pub(crate) const QUALIFIER_PATTERN: &str = r#"(?:"((?:[^"]|"")+)"|([A-Za-z_]\w*))\.$"#; +pub(crate) const TEMPORAL_OPERAND_PATTERN: &str = r"(?i)\A(TIMESTAMP|DATE)\s*'"; +pub(crate) const INTEGER_OPERAND_PATTERN: &str = r"\A-?\d+\z"; +pub(crate) const DECIMAL_OPERAND_PATTERN: &str = r"\A-?\d+\.\d+\z"; +pub(crate) const SQL_TEST_COLUMN_PATTERN: &str = + r"'__(?:expected|ref)__(.+)' names unknown column '([^']+)'"; + +/// Python's `_SEMANTIC_HELP` catalogue. +pub(crate) const SEMANTIC_HELP: &[(&str, &str)] = &[ + ("B003", "qualify this column with the intended input alias"), + ( + "B004", + "give each referenced relation a visible alias in this scope", + ), + ( + "B005", + "match the relation alias list to its available output columns", + ), + ( + "B101", + "check the function spelling or declare the project function before using it", + ), + ( + "B102", + "pass the number of arguments required by this function's signature", + ), + ( + "B210", + "use an expression whose inferred type satisfies this SQL construct", + ), + ("B211", "use a Boolean predicate in this condition"), + ( + "B212", + "use operands supported by this arithmetic operator, or convert them explicitly", + ), + ( + "B213", + "supply argument types supported by this function or conditional expression", + ), + ( + "B214", + "make the assigned expression compatible with the destination type", + ), + ( + "B215", + "align the corresponding types in each set-operation branch", + ), + ( + "B216", + "return the same number of columns on both sides of this operation", + ), + ( + "B217", + "compare compatible types using a correctly typed literal or explicit conversion", + ), + ( + "B218", + "choose a supported source-to-target cast for this dialect", + ), + ( + "B219", + "declare or cast this expression's unknown type so it can be checked", + ), + ( + "B230", + "add this expression to GROUP BY or use it inside an aggregate", + ), + ( + "B231", + "move this aggregate to SELECT or HAVING, or compute it in an input CTE", + ), + ( + "B232", + "use this window function with a valid OVER clause and window context", + ), + ( + "B233", + "use distinct names for relations or CTEs within this scope", + ), + ("B234", "use a valid constant bound for LIMIT or OFFSET"), + ( + "B300", + "change the metadata column name to an existing output column", + ), + ( + "B301", + "align this declared metadata or audit value with the column or argument type", + ), + ( + "B302", + "align the SQL-test fixture or expected columns with the tested resource's output", + ), +]; + +/// Python's `_FINDING_KINDS`: a code test, then the singular and plural nouns. +pub(crate) const FINDING_KINDS: &[(&str, &str, &str)] = &[ + ("^B002$", "unknown column", "unknown columns"), + ("^B003$", "ambiguous column", "ambiguous columns"), + ("^B00[45]$", "reference error", "reference errors"), + ("^B101$", "unknown function", "unknown functions"), + ("^B102$", "function arity error", "function arity errors"), + ("^B216$", "column count mismatch", "column count mismatches"), + ("^B21\\d$", "type mismatch", "type mismatches"), + ( + "^B23[0-2]$", + "grouping or window error", + "grouping or window errors", + ), + ("^B3\\d\\d$", "metadata mismatch", "metadata mismatches"), +]; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/main/complete_semantic_diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_checks/main/complete_semantic_diagnostics.rs new file mode 100644 index 0000000000..ade232f46b --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/main/complete_semantic_diagnostics.rs @@ -0,0 +1,19 @@ +//! Diagnostic recovery, explanations and opt-out rejection after the metadata checks. + +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; + +use crate::semantic_checks::_helpers::completion::complete; +use crate::semantic_checks::models::{CompletionOutcome, CompletionRequest, SemanticDeferral}; +use crate::semantic_validation::models::ProjectCatalog; + +/// Python's recovery, explanation and opt-out rejection, on the compile's analysis pool. +pub fn complete_semantic_diagnostics( + request: &CompletionRequest, + catalog: &ProjectCatalog, +) -> Result { + let pool = catalog.analysis_pool()?; + Ok(pool.install(|| { + catch_compiler_panic(|| Ok(complete(request))) + .unwrap_or(CompletionOutcome::Deferred(SemanticDeferral::NativeFailure)) + })) +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/main/finish_type_recovery.rs b/crates/sqlbuild-analysis/src/semantic_checks/main/finish_type_recovery.rs new file mode 100644 index 0000000000..276e4fd3db --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/main/finish_type_recovery.rs @@ -0,0 +1,16 @@ +//! Python's `recover_output_types` after revalidating the planned models with unknown types. + +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; + +use crate::semantic_checks::_helpers::type_recovery::finish; +use crate::semantic_checks::models::{TypeRecoveryOutcome, TypeRecoveryPlan, TypeRecoveryRequest}; +use crate::semantic_checks::types::RevisedBinding; + +/// Retained diagnostics and model bindings; a native panic is returned as its message. +pub fn finish_type_recovery( + request: &TypeRecoveryRequest, + plan: &TypeRecoveryPlan, + revised: &[Vec], +) -> Result { + catch_compiler_panic(|| Ok(finish(request, plan, revised))) +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/main/mod.rs b/crates/sqlbuild-analysis/src/semantic_checks/main/mod.rs new file mode 100644 index 0000000000..25aba0548b --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/main/mod.rs @@ -0,0 +1,3 @@ +pub mod complete_semantic_diagnostics; +pub mod finish_type_recovery; +pub mod plan_type_recovery; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/main/plan_type_recovery.rs b/crates/sqlbuild-analysis/src/semantic_checks/main/plan_type_recovery.rs new file mode 100644 index 0000000000..8ee38b7c1a --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/main/plan_type_recovery.rs @@ -0,0 +1,19 @@ +//! Python's `recover_output_types` up to the revalidation Python runs on the binding catalog. + +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; + +use crate::semantic_checks::_helpers::type_recovery::plan; +use crate::semantic_checks::models::{SemanticDeferral, TypeRecoveryRequest, TypeRecoveryStep}; +use crate::semantic_validation::models::ProjectCatalog; + +/// Plan type recovery on the compile's analysis pool; a native panic defers to Python. +pub fn plan_type_recovery( + request: &TypeRecoveryRequest, + catalog: &ProjectCatalog, +) -> Result { + let pool = catalog.analysis_pool()?; + Ok(pool.install(|| { + catch_compiler_panic(|| Ok(plan(request))) + .unwrap_or(TypeRecoveryStep::Deferred(SemanticDeferral::NativeFailure)) + })) +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/mod.rs b/crates/sqlbuild-analysis/src/semantic_checks/mod.rs index 85e0e654a1..63bafd480c 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/mod.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/mod.rs @@ -1 +1,10 @@ //! Semantic completion: type recovery, metadata checks, diagnostic recovery and explanations. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/models.rs b/crates/sqlbuild-analysis/src/semantic_checks/models.rs new file mode 100644 index 0000000000..10cd62be2d --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/models.rs @@ -0,0 +1,264 @@ +//! Plain-data requests and outcomes of native semantic completion. + +use std::collections::{HashMap, HashSet}; + +use crate::semantic_checks::types::CompletedParts; + +/// One compiled model output column and the upstream columns it reads. +#[derive(Clone, Debug, Default)] +pub struct LineageOutput { + pub output_column: String, + pub upstream: Vec<(String, String)>, +} + +/// One binding diagnostic a model owns, identified by its value-equality id. +#[derive(Clone, Debug)] +pub struct ModelBinding { + pub id: usize, + pub code: String, + pub message: String, + pub is_error: bool, +} + +/// One raw native binding row Python kept for a model, identified by its value-equality id. +#[derive(Clone, Debug)] +pub struct RawBinding { + pub id: usize, + pub code: String, + pub message: String, + pub start: Option, + pub end: Option, +} + +/// The project-diagnostic fields `update_binding_models` reads. +#[derive(Clone, Debug)] +pub struct DiagnosticOwner { + pub id: usize, + pub code: String, + pub is_model: bool, + pub resource_name: Option, +} + +/// One model as type recovery reads it. +#[derive(Clone, Debug)] +pub struct RecoveryModel { + pub name: String, + pub query_sql: String, + pub inferred_columns: Option>, + pub references: Vec, + pub lineage: Vec, + pub bindings: Vec, + pub raw_bindings: Vec, +} + +/// Python's `recover_output_types` inputs as plain data. +#[derive(Clone, Debug)] +pub struct TypeRecoveryRequest { + pub dialect: Option, + pub models: Vec, + pub diagnostics: Vec, +} + +/// Why native semantic completion hands a stage back to Python. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SemanticDeferral { + UnsupportedDialect, + UnreadableSql, + NonAsciiText, + UnexpectedPosition, + NativeFailure, +} + +impl SemanticDeferral { + /// The deferral kind recorded for the harness. + pub fn as_str(self) -> &'static str { + match self { + Self::UnsupportedDialect => "unsupported_dialect", + Self::UnreadableSql => "unreadable_sql", + Self::NonAsciiText => "non_ascii_text", + Self::UnexpectedPosition => "unexpected_position", + Self::NativeFailure => "native_failure", + } + } +} + +/// The poisoned outputs found before the revalidation Python runs on the binding catalog. +#[derive(Clone, Debug, Default)] +pub struct TypeRecoveryPlan { + pub poisoned: Vec<((String, String), usize)>, + pub roots_by_model: HashMap>, + pub outputs_by_diagnostic: HashMap>, + pub causes_by_model: Vec<(usize, HashSet)>, +} + +impl TypeRecoveryPlan { + /// Failed model indexes to revalidate with unknown poisoned types, in model order. + pub fn revalidated_models(&self) -> Vec { + self.causes_by_model + .iter() + .map(|(index, _)| *index) + .collect() + } + + /// Poisoned `(model, column)` outputs in Python's dict order. + pub fn poisoned_outputs(&self) -> Vec<(String, String)> { + self.poisoned.iter().map(|(key, _)| key.clone()).collect() + } +} + +/// Type recovery's first step: nothing to recover, a plan, or a deferral. +#[derive(Clone, Debug)] +pub enum TypeRecoveryStep { + Unchanged, + Planned(TypeRecoveryPlan), + Deferred(SemanticDeferral), +} + +impl TypeRecoveryStep { + /// `unchanged`, `planned` or `deferred`. + pub fn status(&self) -> &'static str { + match self { + Self::Unchanged => "unchanged", + Self::Planned(_) => "planned", + Self::Deferred(_) => "deferred", + } + } + + /// The deferral kind, when the stage is handed back to Python. + pub fn deferral(&self) -> Option<&'static str> { + match self { + Self::Deferred(deferral) => Some(deferral.as_str()), + _ => None, + } + } + + /// The plan Python's revalidation and the finishing step read. + pub fn plan(&self) -> Option<&TypeRecoveryPlan> { + match self { + Self::Planned(plan) => Some(plan), + _ => None, + } + } +} + +/// Type recovery's edits: retained diagnostics with appended notes and model binding positions. +#[derive(Clone, Debug, Default)] +pub struct TypeRecoveryOutcome { + /// `(project diagnostic index, appended note)` for every retained diagnostic, in order. + pub kept: Vec<(usize, Option)>, + /// Indexes into `kept` of each model's binding diagnostics, in model order. + pub model_bindings: Vec>, +} + +/// A diagnostic location's line and column fields. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct SemanticLocation { + pub line: i64, + pub column: i64, + pub end_line: Option, + pub end_column: Option, +} + +/// One project diagnostic as recovery, explanation and opt-out rejection read it. +#[derive(Clone, Debug)] +pub struct SemanticDiagnostic { + pub id: usize, + pub code: String, + pub message: String, + /// `None` when the diagnostic has no resource type. + pub resource_type: Option, + pub resource_name: Option, + pub line: Option, + pub column: Option, + pub location: Option, + pub help: Option, + pub notes: Vec, +} + +/// One model as recovery, explanation and opt-out rejection read it. +#[derive(Clone, Debug)] +pub struct CompletionModel { + pub name: String, + pub query_sql: String, + pub authored_sql: String, + pub inferred_columns: Vec, + pub lineage: Vec, + pub binding_codes: Vec, + /// The rejected opt-out location's file name, when the model's opt-out is rejected. + pub rejected_opt_out_file: Option, +} + +/// Python's recovery, explanation and opt-out inputs as plain data. +#[derive(Clone, Debug)] +pub struct CompletionRequest { + pub dialect: Option, + pub diagnostics: Vec, + pub models: Vec, + /// Closed relation shapes in Python's dict order. + pub shapes: Vec<(String, Vec<(String, String)>)>, +} + +/// One diagnostic after recovery and explanation, as fields Python replaces. +#[derive(Clone, Debug)] +pub struct CompletedDiagnostic { + pub source: usize, + pub message: String, + pub help: Option, + pub notes: Vec, + pub location: Option, + pub line: Option, + pub column: Option, + pub changed: bool, +} + +/// One P009 error replacing a rejected opt-out model's findings. +#[derive(Clone, Debug)] +pub struct OptOutDiagnostic { + pub model: usize, + pub message: String, + pub note: String, + pub help: String, +} + +/// One entry of the final diagnostic order. +#[derive(Clone, Debug)] +pub enum FinalDiagnostic { + Completed(usize), + OptOut(OptOutDiagnostic), +} + +impl FinalDiagnostic { + /// `(completed position, opt-out error)`: exactly one side is set. + pub fn into_parts(self) -> (Option, Option) { + match self { + Self::Completed(position) => (Some(position), None), + Self::OptOut(opt_out) => (None, Some(opt_out)), + } + } +} + +/// Recovery, explanation and opt-out rejection, or a deferral. +#[derive(Clone, Debug)] +pub enum CompletionOutcome { + Completed { + diagnostics: Vec, + /// Indexes into `diagnostics` per model, or `None` when models keep their bindings. + model_bindings: Option>>, + order: Vec, + }, + Deferred(SemanticDeferral), +} + +impl CompletionOutcome { + /// `(deferral kind, completed parts)`: exactly one side is set. + pub fn into_parts(self) -> (Option<&'static str>, Option) { + match self { + Self::Completed { + diagnostics, + model_bindings, + order, + } => (None, Some((diagnostics, model_bindings, order))), + Self::Deferred(deferral) => (Some(deferral.as_str()), None), + } + } +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests.rs new file mode 100644 index 0000000000..cda52abf5f --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests.rs @@ -0,0 +1,16 @@ +#[path = "tests/test_columns.rs"] +mod columns; +#[path = "tests/test_completion.rs"] +mod completion; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_messages.rs"] +mod messages; +#[path = "tests/test_operand_types.rs"] +mod operand_types; +#[path = "tests/test_parsed_sql.rs"] +mod parsed_sql; +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_type_recovery.rs"] +mod type_recovery; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/helpers.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/helpers.rs new file mode 100644 index 0000000000..37ee96f1c9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/helpers.rs @@ -0,0 +1,499 @@ +use std::collections::BTreeSet; + +use polyglot_sql::{DialectType, SchemaValidationOptions}; + +use crate::semantic_checks::_helpers::messages::{missing_column, sentence_message}; +use crate::semantic_checks::_helpers::parsed_sql::parsed_model_facts; +use crate::semantic_checks::main::complete_semantic_diagnostics::complete_semantic_diagnostics; +use crate::semantic_checks::main::finish_type_recovery::finish_type_recovery; +use crate::semantic_checks::main::plan_type_recovery::plan_type_recovery; +use crate::semantic_checks::models::{ + CompletedDiagnostic, CompletionModel, CompletionRequest, DiagnosticOwner, FinalDiagnostic, + LineageOutput, ModelBinding, RawBinding, RecoveryModel, SemanticDeferral, SemanticDiagnostic, + SemanticLocation, TypeRecoveryPlan, TypeRecoveryRequest, +}; +use crate::semantic_checks::tests::test_types::{ + CompletionTestCase, DescribedDiagnostic, OperandTypeTestCase, TypeRecoveryTestCase, +}; +use crate::semantic_validation::models::ProjectCatalog; + +/// What type recovery decided: status, poisoned outputs, revalidated models, kept and bindings. +pub(crate) type RecoverySummary = ( + &'static str, + Vec<(String, String)>, + Vec, + Vec<(usize, Option)>, + Vec>, +); + +/// What completion decided: deferral, final diagnostics and model binding positions. +pub(crate) type CompletionSummary = ( + Option<&'static str>, + Vec, + Option>>, +); + +pub(crate) fn strings(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +pub(crate) fn shape(columns: &[(&str, &str)]) -> Vec<(String, String)> { + columns + .iter() + .map(|(name, kind)| ((*name).to_owned(), (*kind).to_owned())) + .collect() +} + +pub(crate) fn pairs(values: &[(&str, &str)]) -> BTreeSet<(String, String)> { + shape(values).into_iter().collect() +} + +pub(crate) fn names(values: &[&str]) -> BTreeSet { + strings(values).into_iter().collect() +} + +pub(crate) fn sentence(message: &str) -> Result { + sentence_message(message) +} + +/// The missing column and table Python reads from a message. +pub(crate) fn missing_parts( + message: &str, +) -> Result)>, SemanticDeferral> { + missing_column(message) +} + +/// An expected missing column and table as owned strings. +pub(crate) fn owned_missing( + expected: Result)>, SemanticDeferral>, +) -> Result)>, SemanticDeferral> { + expected.map(|missing| missing.map(owned_parts)) +} + +fn owned_parts((column, table): (&str, Option<&str>)) -> (String, Option) { + (column.to_owned(), table.map(str::to_owned)) +} + +/// Aliases and unaliased outputs of one DuckDB model. +pub(crate) fn parsed_facts(sql: &str) -> (BTreeSet<(String, String)>, BTreeSet) { + let facts = parsed_model_facts(sql, Some("duckdb")).expect("duckdb parses"); + ( + facts.aliases.into_iter().collect(), + facts.unaliased_outputs.into_iter().collect(), + ) +} + +pub(crate) fn catalog() -> ProjectCatalog { + ProjectCatalog::with_options( + DialectType::DuckDB, + SchemaValidationOptions::default(), + false, + ) +} + +pub(crate) fn lineage(output: &str, upstream: &[(&str, &str)]) -> LineageOutput { + LineageOutput { + output_column: output.to_owned(), + upstream: shape(upstream), + } +} + +fn recovery_model( + name: &str, + query_sql: &str, + references: &[&str], + lineage: Vec, + binding: Option<(usize, &str, i64, i64)>, + is_error: bool, +) -> RecoveryModel { + let message = |code: &str| format!("{code} failed"); + RecoveryModel { + name: name.to_owned(), + query_sql: query_sql.to_owned(), + inferred_columns: Some( + lineage + .iter() + .map(|output| output.output_column.clone()) + .collect(), + ), + references: strings(references), + lineage, + bindings: binding + .iter() + .map(|(id, code, _, _)| ModelBinding { + id: *id, + code: (*code).to_owned(), + message: message(code), + is_error, + }) + .collect(), + raw_bindings: binding + .iter() + .map(|(id, code, start, end)| RawBinding { + id: *id, + code: (*code).to_owned(), + message: message(code), + start: Some(*start), + end: Some(*end), + }) + .collect(), + } +} + +fn owner(id: usize, code: &str, model: &str) -> DiagnosticOwner { + DiagnosticOwner { + id, + code: code.to_owned(), + is_model: true, + resource_name: Some(model.to_owned()), + } +} + +/// A staging projection type error poisoning a mart output and a report reading it. +fn poisoned_chain(dialect: Option<&str>, is_error: bool) -> TypeRecoveryRequest { + TypeRecoveryRequest { + dialect: dialect.map(str::to_owned), + models: vec![ + recovery_model( + "stg", + "SELECT order_id, amount + 'x' AS amount FROM __source(\"raw_orders\")", + &["raw_orders"], + vec![ + lineage("order_id", &[("raw_orders", "order_id")]), + lineage("amount", &[("raw_orders", "amount")]), + ], + Some((0, "B212", 17, 29)), + is_error, + ), + recovery_model( + "mart", + "SELECT customer_id, SUM(amount) AS total FROM __ref(\"stg\") WHERE amount > 5 GROUP BY 1", + &["stg"], + vec![ + lineage("customer_id", &[("stg", "customer_id")]), + lineage("total", &[("stg", "amount")]), + ], + Some((1, "B217", 56, 62)), + is_error, + ), + recovery_model( + "report", + "SELECT total AS x FROM __ref(\"mart\")", + &["mart"], + vec![lineage("x", &[("mart", "total")])], + None, + is_error, + ), + ], + diagnostics: vec![owner(0, "B212", "stg"), owner(1, "B217", "mart")], + } +} + +/// Plan and finish type recovery for one case of the poisoned chain. +pub(crate) fn recovery_summary(test_case: &TypeRecoveryTestCase) -> RecoverySummary { + let request = poisoned_chain(test_case.dialect, test_case.errors); + let step = plan_type_recovery(&request, &catalog()).expect("the analysis pool runs"); + let revised = vec![ + test_case + .revised + .iter() + .map(|(code, start, end)| ((*code).to_owned(), *start, *end)) + .collect(), + ]; + let outcome = step + .plan() + .map(|plan| finish_type_recovery(&request, plan, &revised).expect("finishes")) + .unwrap_or_default(); + ( + step.deferral().unwrap_or(step.status()), + step.plan() + .map(TypeRecoveryPlan::poisoned_outputs) + .unwrap_or_default(), + step.plan() + .map(TypeRecoveryPlan::revalidated_models) + .unwrap_or_default(), + outcome.kept, + outcome.model_bindings, + ) +} + +/// Expected poisoned outputs as owned pairs, in order. +pub(crate) fn expected_poisoned(values: &[(&str, &str)]) -> Vec<(String, String)> { + shape(values) +} + +/// Expected retained diagnostics with owned notes. +pub(crate) fn expected_kept(values: &[(usize, Option<&str>)]) -> Vec<(usize, Option)> { + values + .iter() + .map(|(index, note)| (*index, note.map(str::to_owned))) + .collect() +} + +/// Expected model binding positions as owned vectors. +pub(crate) fn expected_bindings(values: &[&[usize]]) -> Vec> { + values.iter().map(|positions| positions.to_vec()).collect() +} + +/// A model diagnostic located at `(line, column)`. +fn model_diagnostic( + id: usize, + code: &str, + message: &str, + model: &str, + location: Option<(i64, i64)>, +) -> SemanticDiagnostic { + SemanticDiagnostic { + id, + code: code.to_owned(), + message: message.to_owned(), + resource_type: Some("model".to_owned()), + resource_name: Some(model.to_owned()), + line: location.map(|(line, _)| line), + column: location.map(|(_, column)| column), + location: location.map(|(line, column)| SemanticLocation { + line, + column, + end_line: None, + end_column: None, + }), + help: None, + notes: Vec::new(), + } +} + +/// A model with its authored SQL behind a MODEL header. +fn completion_model( + name: &str, + query_sql: &str, + inferred: &[&str], + lineage: Vec, + binding_codes: &[&str], +) -> CompletionModel { + CompletionModel { + name: name.to_owned(), + query_sql: query_sql.to_owned(), + authored_sql: format!("MODEL (\n description \"{name}\",\n);\n\n{query_sql}\n"), + inferred_columns: strings(inferred), + lineage, + binding_codes: strings(binding_codes), + rejected_opt_out_file: None, + } +} + +/// A staging typo poisoning a mart use, a temporal comparison, a rejected opt-out and a test error. +fn project_request() -> CompletionRequest { + let mut legacy = completion_model( + "legacy", + "SELECT mystery(order_id) AS m FROM __ref(\"stg\")", + &["m"], + Vec::new(), + &["B101"], + ); + legacy.rejected_opt_out_file = Some("legacy.sql".to_owned()); + let mut sql_test = model_diagnostic( + 4, + "B002", + "Unknown column 'x' in table 'stg'", + "report", + None, + ); + sql_test.resource_type = Some("sql_test".to_owned()); + CompletionRequest { + dialect: Some("duckdb".to_owned()), + diagnostics: vec![ + model_diagnostic( + 0, + "B002", + "Unknown column 'amonut' in table 'raw_orders'", + "stg", + Some((5, 18)), + ), + model_diagnostic( + 1, + "B002", + "Unknown column 'amount' in table 'stg'", + "mart", + Some((5, 24)), + ), + model_diagnostic( + 2, + "B217", + "Cannot compare timestamp and integer (context: o.ordered_at > 5)", + "mart", + Some((7, 7)), + ), + model_diagnostic( + 3, + "B101", + "Unknown function 'mystery'", + "legacy", + Some((5, 8)), + ), + sql_test, + ], + models: vec![ + completion_model( + "stg", + "SELECT order_id, amonut, ordered_at FROM __source(\"raw_orders\")", + &["order_id", "amonut", "ordered_at"], + vec![lineage("amonut", &[("raw_orders", "amonut")])], + &["B002"], + ), + completion_model( + "mart", + "SELECT customer_id, SUM(amount) AS total\nFROM __ref(\"stg\") AS o\n\ + WHERE o.ordered_at > 5\nGROUP BY 1", + &["customer_id", "total"], + vec![lineage("total", &[("stg", "amount")])], + &["B002", "B217"], + ), + legacy, + ], + shapes: vec![ + ( + "raw_orders".to_owned(), + shape(&[ + ("order_id", "INTEGER"), + ("customer_id", "INTEGER"), + ("amount", "DOUBLE"), + ("status", "VARCHAR"), + ]), + ), + ( + "stg".to_owned(), + shape(&[ + ("order_id", "INTEGER"), + ("amount", "UNKNOWN"), + ("ordered_at", "TIMESTAMP"), + ]), + ), + ], + } +} + +/// One expected diagnostic of the completion project. +pub(crate) fn described( + code: &str, + message: &str, + help: Option<&str>, + notes: &[&str], + location: Option<(i64, i64, Option, Option)>, +) -> DescribedDiagnostic { + ( + code.to_owned(), + message.to_owned(), + help.map(str::to_owned), + strings(notes), + location, + ) +} + +/// Complete one variant of the project request. +pub(crate) fn completion_summary(test_case: &CompletionTestCase) -> CompletionSummary { + let mut request = project_request(); + request.models[1] + .authored_sql + .push_str(["", "-- caf\u{e9}\n"][usize::from(test_case.non_ascii_comment)]); + let kept_diagnostics = + [request.diagnostics.len(), 0][usize::from(test_case.without_diagnostics)]; + request.diagnostics.truncate(kept_diagnostics); + request.models[2].rejected_opt_out_file = + [Some("legacy.sql".to_owned()), None][usize::from(test_case.without_diagnostics)].clone(); + let outcome = complete_semantic_diagnostics(&request, &catalog()).expect("the pool runs"); + let (deferral, parts) = outcome.into_parts(); + let (diagnostics, model_bindings, order) = parts.unwrap_or_default(); + let described_order: Vec = order + .into_iter() + .map(|entry| described_entry(&request, &diagnostics, entry)) + .collect(); + (deferral, described_order, model_bindings) +} + +fn described_entry( + request: &CompletionRequest, + diagnostics: &[CompletedDiagnostic], + entry: FinalDiagnostic, +) -> DescribedDiagnostic { + let (position, opt_out) = entry.into_parts(); + let completed = position.map(|position| described_completed(request, &diagnostics[position])); + let rejected = opt_out.map(|opt_out| { + ( + "P009".to_owned(), + opt_out.message, + Some(opt_out.help), + vec![opt_out.note], + None, + ) + }); + completed + .or(rejected) + .expect("one side of an order entry is set") +} + +fn described_completed( + request: &CompletionRequest, + diagnostic: &CompletedDiagnostic, +) -> DescribedDiagnostic { + ( + request.diagnostics[diagnostic.source].code.clone(), + diagnostic.message.clone(), + diagnostic.help.clone(), + diagnostic.notes.clone(), + diagnostic.location.map(|location| { + ( + location.line, + location.column, + location.end_line, + location.end_column, + ) + }), + ) +} + +/// Explain a B212 and a B217 on unqualified operands of a mart reading `stg`. +pub(crate) fn operand_type_summary(test_case: &OperandTypeTestCase) -> CompletionSummary { + let query_sql = format!( + "SELECT order_id, ordered_at + 1 AS shifted\nFROM __ref(\"stg\"){}\nWHERE ordered_at > 5", + test_case.joined + ); + let request = CompletionRequest { + dialect: Some("duckdb".to_owned()), + diagnostics: vec![ + model_diagnostic( + 0, + "B212", + "Arithmetic operation expects NUMERIC-compatible operands, found TIMESTAMP and INTEGER", + "mart", + Some((5, 18)), + ), + model_diagnostic( + 1, + "B217", + "Incompatible comparison between TIMESTAMP and INTEGER", + "mart", + Some((7, 7)), + ), + ], + models: vec![completion_model( + "mart", + &query_sql, + &["order_id", "shifted"], + Vec::new(), + &["B212", "B217"], + )], + shapes: test_case + .shapes + .iter() + .map(|(name, columns)| ((*name).to_owned(), shape(columns))) + .collect(), + }; + let outcome = complete_semantic_diagnostics(&request, &catalog()).expect("the pool runs"); + let (deferral, parts) = outcome.into_parts(); + let (diagnostics, model_bindings, order) = parts.unwrap_or_default(); + let described_order: Vec = order + .into_iter() + .map(|entry| described_entry(&request, &diagnostics, entry)) + .collect(); + (deferral, described_order, model_bindings) +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_columns.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_columns.rs new file mode 100644 index 0000000000..ae5aecd7ca --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_columns.rs @@ -0,0 +1,67 @@ +use crate::semantic_checks::_helpers::columns::{closest_column, ordered_columns}; +use crate::semantic_checks::models::SemanticDeferral; +use crate::semantic_checks::tests::helpers::shape; +use crate::semantic_checks::tests::test_types::ClosestColumnTestCase; + +const ORDER_COLUMNS: &[(&str, &str)] = &[ + ("order_id", "INTEGER"), + ("customer_id", "INTEGER"), + ("amount", "DOUBLE"), + ("status", "VARCHAR"), +]; + +#[test] +fn given_unknown_column_names_when_suggesting_then_matches_python_order_and_choice() { + let test_cases = [ + ClosestColumnTestCase { + description: "a transposition within two edits", + name: "amonut", + columns: ORDER_COLUMNS, + expected_closest: Ok(Some("amount")), + expected_order: Ok(&["amount", "status", "customer_id", "order_id"]), + }, + ClosestColumnTestCase { + description: "an abbreviation keeping the first and last letters", + name: "cust_id", + columns: ORDER_COLUMNS, + expected_closest: Ok(Some("customer_id")), + expected_order: Ok(&["customer_id", "order_id", "status", "amount"]), + }, + ClosestColumnTestCase { + description: "a different case", + name: "Amount", + columns: ORDER_COLUMNS, + expected_closest: Ok(Some("amount")), + expected_order: Ok(&["amount", "customer_id", "order_id", "status"]), + }, + ClosestColumnTestCase { + description: "no close column", + name: "zzz", + columns: ORDER_COLUMNS, + expected_closest: Ok(None), + expected_order: Ok(&["amount", "customer_id", "order_id", "status"]), + }, + ClosestColumnTestCase { + description: "a column Python case-folds beyond ASCII", + name: "strasse", + columns: &[("stra\u{df}e", "VARCHAR")], + expected_closest: Err(SemanticDeferral::NonAsciiText), + expected_order: Err(SemanticDeferral::NonAsciiText), + }, + ]; + for test_case in test_cases { + let columns = shape(test_case.columns); + assert_eq!( + closest_column(test_case.name, &columns), + test_case.expected_closest, + "{}", + test_case.description + ); + assert_eq!( + ordered_columns(test_case.name, &columns), + test_case.expected_order.map(<[&str]>::to_vec), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_completion.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_completion.rs new file mode 100644 index 0000000000..3d21831efe --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_completion.rs @@ -0,0 +1,78 @@ +use crate::semantic_checks::tests::helpers::{completion_summary, described}; +use crate::semantic_checks::tests::test_types::CompletionTestCase; + +const OPT_OUT_HELP: &str = "remove `sql_analysis false` and fix the findings it was hiding: \ + 1 unknown function (run `sqb compile` to see them)\n = help: to allow `sql_analysis false` \ + on any model, SQL test or audit, set this in sqlbuild_project.toml:\n \ + [settings]\n require_sql_analysis = false"; +const OPT_OUT_NOTE: &str = "sqlbuild_project.toml sets [settings] require_sql_analysis = true, \ + which only allows `sql_analysis false` on SQL that SQLBuild cannot parse; this model parses \ + successfully."; + +#[test] +fn given_failing_project_when_completing_then_recovers_explains_and_rejects_like_python() { + let test_cases = [ + CompletionTestCase { + description: "a recovered typo, an explained comparison, a test error and an opt-out", + non_ascii_comment: false, + without_diagnostics: false, + expected_deferral: None, + expected_diagnostics: vec![ + described( + "B002", + "Unknown column 'amonut' in raw_orders", + Some("did you mean 'amount'?"), + &[ + "raw_orders has: amount, status, customer_id, order_id", + "1 downstream uses of stg.amount were not checked because of this error", + ], + Some((5, 18, None, None)), + ), + described( + "B217", + "Cannot compare TIMESTAMP and INTEGER", + Some("compare with a timestamp, for example TIMESTAMP '2026-04-01'"), + &["o.ordered_at is TIMESTAMP, 5 is INTEGER"], + Some((7, 7, Some(7), Some(23))), + ), + described("B002", "Unknown column 'x' in table 'stg'", None, &[], None), + described( + "P009", + "`sql_analysis false` is not needed for model 'legacy'", + Some(OPT_OUT_HELP), + &[OPT_OUT_NOTE], + None, + ), + ], + expected_bindings: Some(vec![vec![0], vec![1], vec![2]]), + }, + CompletionTestCase { + description: "non-ASCII authored SQL in an explained model defers to Python", + non_ascii_comment: true, + without_diagnostics: false, + expected_deferral: Some("non_ascii_text"), + expected_diagnostics: Vec::new(), + expected_bindings: None, + }, + CompletionTestCase { + description: "no diagnostics keep every model's bindings", + non_ascii_comment: false, + without_diagnostics: true, + expected_deferral: None, + expected_diagnostics: Vec::new(), + expected_bindings: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + completion_summary(&test_case), + ( + test_case.expected_deferral, + test_case.expected_diagnostics.clone(), + test_case.expected_bindings.clone(), + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_messages.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_messages.rs new file mode 100644 index 0000000000..b0769a0568 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_messages.rs @@ -0,0 +1,103 @@ +use crate::semantic_checks::_helpers::messages::comparison_help; +use crate::semantic_checks::models::SemanticDeferral; +use crate::semantic_checks::tests::helpers::{missing_parts, owned_missing, sentence}; +use crate::semantic_checks::tests::test_types::{ComparisonHelpTestCase, SentenceMessageTestCase}; + +#[test] +fn given_binding_messages_when_rewriting_then_matches_python_sentences() { + let test_cases = [ + SentenceMessageTestCase { + description: "a context suffix and unquoted type words", + message: "Binary operator types integer and timestamp are incompatible (context: x > 5)", + expected_sentence: Ok("BINARY operator types INTEGER and TIMESTAMP are incompatible"), + expected_missing: Ok(None), + }, + SentenceMessageTestCase { + description: "quoted type words stay as written", + message: "Cannot compare 'varchar' with date values", + expected_sentence: Ok("Cannot compare 'varchar' with DATE values"), + expected_missing: Ok(None), + }, + SentenceMessageTestCase { + description: "an unknown column in a named table", + message: "Unknown column 'amonut' in table 'raw_orders'", + expected_sentence: Ok("Unknown column 'amonut' in table 'raw_orders'"), + expected_missing: Ok(Some(("amonut", Some("raw_orders")))), + }, + SentenceMessageTestCase { + description: "an unknown column without a table", + message: "Unknown column 'amonut'", + expected_sentence: Ok("Unknown column 'amonut'"), + expected_missing: Ok(Some(("amonut", None))), + }, + SentenceMessageTestCase { + description: "a non-ASCII column Python's regex classes read differently", + message: "Unknown column 'stra\u{df}e'", + expected_sentence: Err(SemanticDeferral::NonAsciiText), + expected_missing: Err(SemanticDeferral::NonAsciiText), + }, + SentenceMessageTestCase { + description: "a line break Python's end anchor treats specially", + message: "Unknown column 'a'\nin table 'b'", + expected_sentence: Err(SemanticDeferral::NonAsciiText), + expected_missing: Err(SemanticDeferral::NonAsciiText), + }, + ]; + for test_case in test_cases { + assert_eq!( + sentence(test_case.message), + test_case.expected_sentence.map(str::to_owned), + "{}", + test_case.description + ); + assert_eq!( + missing_parts(test_case.message), + owned_missing(test_case.expected_missing), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_comparison_types_when_building_help_then_offers_python_literals() { + let test_cases = [ + ComparisonHelpTestCase { + description: "a timestamp on BigQuery", + types: &["TIMESTAMP", "INTEGER"], + dialect: Some("bigquery"), + expected_help: "compare with a timestamp, for example TIMESTAMP '2026-04-01 00:00:00+00'", + }, + ComparisonHelpTestCase { + description: "a timestamp elsewhere", + types: &["INTEGER", "TIMESTAMP WITH TIME ZONE"], + dialect: Some("duckdb"), + expected_help: "compare with a timestamp, for example TIMESTAMP '2026-04-01'", + }, + ComparisonHelpTestCase { + description: "a date", + types: &["DATE"], + dialect: None, + expected_help: "compare with a date, for example DATE '2026-04-01'", + }, + ComparisonHelpTestCase { + description: "no temporal type", + types: &["INTEGER", "VARCHAR"], + dialect: None, + expected_help: "compare compatible types using a correctly typed literal or explicit conversion", + }, + ]; + for test_case in test_cases { + let types: Vec = test_case + .types + .iter() + .map(|value| (*value).to_owned()) + .collect(); + assert_eq!( + comparison_help(&types, test_case.dialect), + test_case.expected_help, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_operand_types.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_operand_types.rs new file mode 100644 index 0000000000..15f522e62d --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_operand_types.rs @@ -0,0 +1,96 @@ +use crate::semantic_checks::tests::helpers::{described, operand_type_summary}; +use crate::semantic_checks::tests::test_types::OperandTypeTestCase; + +const ARITHMETIC_MESSAGE: &str = + "Arithmetic operation expects NUMERIC-compatible operands, found TIMESTAMP and INTEGER"; +const ARITHMETIC_HELP: &str = + "use operands supported by this arithmetic operator, or convert them explicitly"; +const COMPARISON_MESSAGE: &str = "Incompatible comparison between TIMESTAMP and INTEGER"; +const TIMESTAMP_HELP: &str = "compare with a timestamp, for example TIMESTAMP '2026-04-01'"; + +#[test] +fn given_unqualified_operand_types_when_explaining_then_notes_only_known_types_like_python() { + let test_cases = [ + OperandTypeTestCase { + description: "an empty column type is no type, so neither error gets a note", + joined: "", + shapes: &[("stg", &[("order_id", "INTEGER"), ("ordered_at", "")])], + expected_diagnostics: vec![ + described( + "B212", + ARITHMETIC_MESSAGE, + Some(ARITHMETIC_HELP), + &[], + Some((5, 18, Some(5), Some(32))), + ), + described( + "B217", + COMPARISON_MESSAGE, + Some(TIMESTAMP_HELP), + &[], + Some((7, 7, Some(7), Some(21))), + ), + ], + }, + OperandTypeTestCase { + description: "an empty and an INTEGER type across two inputs stay ambiguous", + joined: " JOIN __ref(\"orders\") ON TRUE", + shapes: &[ + ("stg", &[("order_id", "INTEGER"), ("ordered_at", "")]), + ("orders", &[("ordered_at", "INTEGER")]), + ], + expected_diagnostics: vec![ + described( + "B212", + ARITHMETIC_MESSAGE, + Some(ARITHMETIC_HELP), + &[], + Some((5, 18, Some(5), Some(32))), + ), + described( + "B217", + COMPARISON_MESSAGE, + Some(TIMESTAMP_HELP), + &[], + Some((7, 7, Some(7), Some(21))), + ), + ], + }, + OperandTypeTestCase { + description: "a known type is noted for both errors", + joined: "", + shapes: &[( + "stg", + &[("order_id", "INTEGER"), ("ordered_at", "timestamp")], + )], + expected_diagnostics: vec![ + described( + "B212", + ARITHMETIC_MESSAGE, + Some(ARITHMETIC_HELP), + &["ordered_at is TIMESTAMP, 1 is INTEGER"], + Some((5, 18, Some(5), Some(32))), + ), + described( + "B217", + COMPARISON_MESSAGE, + Some(TIMESTAMP_HELP), + &["ordered_at is TIMESTAMP, 5 is INTEGER"], + Some((7, 7, Some(7), Some(21))), + ), + ], + }, + ]; + for test_case in test_cases { + assert_eq!( + operand_type_summary(&test_case), + ( + None, + test_case.expected_diagnostics.clone(), + Some(vec![vec![0, 1]]) + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_parsed_sql.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_parsed_sql.rs new file mode 100644 index 0000000000..e09cd235ce --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_parsed_sql.rs @@ -0,0 +1,106 @@ +use crate::semantic_checks::_helpers::parsed_sql::projection_spans; +use crate::semantic_checks::models::SemanticDeferral; +use crate::semantic_checks::tests::helpers::{names, pairs, parsed_facts}; +use crate::semantic_checks::tests::test_types::{ParsedFactsTestCase, ProjectionSpanTestCase}; + +#[test] +fn given_select_lists_when_finding_projection_spans_then_matches_python_tokens() { + let test_cases = [ + ProjectionSpanTestCase { + description: "aliases and a function call up to FROM", + sql: "SELECT a, b AS c, f(x, y) FROM t", + dialect: Some("duckdb"), + expected_spans: Ok(&[(6, 8), (9, 16), (17, 26)]), + }, + ProjectionSpanTestCase { + description: "a nested select list ends at WHERE", + sql: "SELECT a, (SELECT 1, 2) AS s\nWHERE 1 = 1", + dialect: Some("duckdb"), + expected_spans: Ok(&[(6, 8), (9, 29)]), + }, + ProjectionSpanTestCase { + description: "a select list running to the end", + sql: "SELECT a, b", + dialect: Some("duckdb"), + expected_spans: Ok(&[(6, 8), (9, 11)]), + }, + ProjectionSpanTestCase { + description: "a CTE before the top-level select", + sql: "WITH c AS (SELECT 1 AS x) SELECT x, y FROM c", + dialect: Some("duckdb"), + expected_spans: Ok(&[(32, 34), (35, 38)]), + }, + ProjectionSpanTestCase { + description: "code-point offsets after a non-ASCII literal", + sql: "SELECT '\u{e9}', b FROM t", + dialect: Some("duckdb"), + expected_spans: Ok(&[(6, 10), (11, 14)]), + }, + ProjectionSpanTestCase { + description: "no dialect, which the wheel's tokenizer rejects", + sql: "SELECT a", + dialect: None, + expected_spans: Err(SemanticDeferral::UnsupportedDialect), + }, + ProjectionSpanTestCase { + description: "a dialect outside the native parser build", + sql: "SELECT a", + dialect: Some("mysql"), + expected_spans: Err(SemanticDeferral::UnsupportedDialect), + }, + ]; + for test_case in test_cases { + assert_eq!( + projection_spans(test_case.sql, test_case.dialect), + test_case.expected_spans.map(<[(usize, usize)]>::to_vec), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_model_sql_when_parsing_then_finds_python_aliases_and_unaliased_outputs() { + let test_cases = [ + ParsedFactsTestCase { + description: "aliases, a shared alias name and an aliased projection", + sql: "SELECT o.order_id, amount, c.name AS customer FROM __ref(\"orders\") AS o \ + JOIN __ref(\"customers\") c ON TRUE JOIN __ref(\"orders\") AS c2 ON TRUE", + expected_aliases: &[ + ("c", "customers"), + ("c2", "orders"), + ("customers", "customers"), + ("o", "orders"), + ("orders", "orders"), + ], + expected_unaliased: &["amount", "order_id"], + }, + ParsedFactsTestCase { + description: "a set operation has no top-level select list", + sql: "SELECT a FROM __source(\"raw_orders\") UNION ALL SELECT b FROM t", + expected_aliases: &[("raw_orders", "raw_orders"), ("t", "t")], + expected_unaliased: &[], + }, + ParsedFactsTestCase { + description: "SQL the wheel cannot parse", + sql: "SELECT FROM (", + expected_aliases: &[], + expected_unaliased: &[], + }, + ]; + for test_case in test_cases { + let (aliases, unaliased) = parsed_facts(test_case.sql); + assert_eq!( + aliases, + pairs(test_case.expected_aliases), + "{}", + test_case.description + ); + assert_eq!( + unaliased, + names(test_case.expected_unaliased), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_type_recovery.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_type_recovery.rs new file mode 100644 index 0000000000..e96abd40b0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_type_recovery.rs @@ -0,0 +1,89 @@ +use crate::semantic_checks::tests::helpers::{ + expected_bindings, expected_kept, expected_poisoned, recovery_summary, +}; +use crate::semantic_checks::tests::test_types::TypeRecoveryTestCase; + +const DOWNSTREAM_NOTE: &str = + "2 downstream output uses were not type-checked because of this error"; +const POISONED: &[(&str, &str)] = &[("stg", "amount"), ("mart", "total"), ("report", "x")]; + +#[test] +fn given_poisoned_projection_when_recovering_types_then_counts_and_drops_like_python() { + let test_cases = [ + TypeRecoveryTestCase { + description: "a downstream error the unknown type explains is dropped", + dialect: Some("duckdb"), + errors: true, + revised: &[], + expected_status: "planned", + expected_poisoned: POISONED, + expected_revalidated: &[1], + expected_kept: &[(0, Some(DOWNSTREAM_NOTE))], + expected_bindings: &[&[0], &[], &[]], + }, + TypeRecoveryTestCase { + description: "a downstream error the revalidation still reports is kept", + dialect: Some("duckdb"), + errors: true, + revised: &[("B217", Some(56), Some(62))], + expected_status: "planned", + expected_poisoned: POISONED, + expected_revalidated: &[1], + expected_kept: &[(0, Some(DOWNSTREAM_NOTE)), (1, None)], + expected_bindings: &[&[0], &[1], &[]], + }, + TypeRecoveryTestCase { + description: "no dialect, which the wheel's tokenizer rejects", + dialect: None, + errors: true, + revised: &[], + expected_status: "unsupported_dialect", + expected_poisoned: &[], + expected_revalidated: &[], + expected_kept: &[], + expected_bindings: &[], + }, + TypeRecoveryTestCase { + description: "only warnings leave the project unchanged", + dialect: Some("duckdb"), + errors: false, + revised: &[], + expected_status: "unchanged", + expected_poisoned: &[], + expected_revalidated: &[], + expected_kept: &[], + expected_bindings: &[], + }, + ]; + for test_case in test_cases { + let (status, poisoned, revalidated, kept, bindings) = recovery_summary(&test_case); + assert_eq!( + status, test_case.expected_status, + "{}", + test_case.description + ); + assert_eq!( + poisoned, + expected_poisoned(test_case.expected_poisoned), + "{}", + test_case.description + ); + assert_eq!( + revalidated, test_case.expected_revalidated, + "{}", + test_case.description + ); + assert_eq!( + kept, + expected_kept(test_case.expected_kept), + "{}", + test_case.description + ); + assert_eq!( + bindings, + expected_bindings(test_case.expected_bindings), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_types.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_types.rs new file mode 100644 index 0000000000..b8f0a329c8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_types.rs @@ -0,0 +1,75 @@ +use crate::semantic_checks::models::SemanticDeferral; + +/// One located `(code, message, help, notes, location)` diagnostic of a completed project. +pub(crate) type DescribedDiagnostic = ( + String, + String, + Option, + Vec, + Option<(i64, i64, Option, Option)>, +); + +pub(crate) struct ProjectionSpanTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) dialect: Option<&'static str>, + pub(crate) expected_spans: Result<&'static [(usize, usize)], SemanticDeferral>, +} + +pub(crate) struct ParsedFactsTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_aliases: &'static [(&'static str, &'static str)], + pub(crate) expected_unaliased: &'static [&'static str], +} + +pub(crate) struct ClosestColumnTestCase { + pub(crate) description: &'static str, + pub(crate) name: &'static str, + pub(crate) columns: &'static [(&'static str, &'static str)], + pub(crate) expected_closest: Result, SemanticDeferral>, + pub(crate) expected_order: Result<&'static [&'static str], SemanticDeferral>, +} + +pub(crate) struct SentenceMessageTestCase { + pub(crate) description: &'static str, + pub(crate) message: &'static str, + pub(crate) expected_sentence: Result<&'static str, SemanticDeferral>, + pub(crate) expected_missing: + Result)>, SemanticDeferral>, +} + +pub(crate) struct ComparisonHelpTestCase { + pub(crate) description: &'static str, + pub(crate) types: &'static [&'static str], + pub(crate) dialect: Option<&'static str>, + pub(crate) expected_help: &'static str, +} + +pub(crate) struct TypeRecoveryTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: Option<&'static str>, + pub(crate) errors: bool, + pub(crate) revised: &'static [(&'static str, Option, Option)], + pub(crate) expected_status: &'static str, + pub(crate) expected_poisoned: &'static [(&'static str, &'static str)], + pub(crate) expected_revalidated: &'static [usize], + pub(crate) expected_kept: &'static [(usize, Option<&'static str>)], + pub(crate) expected_bindings: &'static [&'static [usize]], +} + +pub(crate) struct CompletionTestCase { + pub(crate) description: &'static str, + pub(crate) non_ascii_comment: bool, + pub(crate) without_diagnostics: bool, + pub(crate) expected_deferral: Option<&'static str>, + pub(crate) expected_diagnostics: Vec, + pub(crate) expected_bindings: Option>>, +} + +pub(crate) struct OperandTypeTestCase { + pub(crate) description: &'static str, + pub(crate) joined: &'static str, + pub(crate) shapes: &'static [(&'static str, &'static [(&'static str, &'static str)])], + pub(crate) expected_diagnostics: Vec, +} diff --git a/crates/sqlbuild-analysis/src/semantic_checks/types.rs b/crates/sqlbuild-analysis/src/semantic_checks/types.rs new file mode 100644 index 0000000000..445dc1ef9f --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/types.rs @@ -0,0 +1,13 @@ +//! Row aliases shared by the semantic completion entries and their callers. + +use crate::semantic_checks::models::{CompletedDiagnostic, FinalDiagnostic}; + +/// One revalidated binding row: `(code, start, end)`. +pub type RevisedBinding = (String, Option, Option); + +/// A completed outcome's diagnostics, model binding positions and final order. +pub type CompletedParts = ( + Vec, + Option>>, + Vec, +); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs new file mode 100644 index 0000000000..c9ac573eaa --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs @@ -0,0 +1,173 @@ +//! Native diagnostic recovery, explanations and opt-out rejection. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyRef, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::semantic_checks::main::complete_semantic_diagnostics::complete_semantic_diagnostics; +use sqlbuild_analysis::semantic_checks::models::{ + CompletedDiagnostic, CompletionModel, CompletionRequest, FinalDiagnostic, SemanticDiagnostic, + SemanticLocation, +}; + +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::semantic_checks::type_recovery::{LineageInput, lineage_outputs}; +use crate::bindings::models::ProjectCatalog; +use crate::bindings::types::CompilerDetach; + +/// `(line, column, end_line, end_column)`. +type LocationRow = (i64, i64, Option, Option); +/// `(id, code, message, resource_type, resource_name, line, column, location, help, notes)`. +type DiagnosticInput = ( + usize, + String, + String, + Option, + Option, + Option, + Option, + Option, + Option, + Vec, +); +/// `(name, query_sql, authored_sql, inferred names, lineage, binding codes, opt-out file)`. +type ModelInput = ( + String, + String, + String, + Vec, + Vec, + Vec, + Option, +); +/// `(dialect, diagnostics, models, shapes)`. +type RequestInput = ( + Option, + Vec, + Vec, + Vec<(String, Vec<(String, String)>)>, +); +/// `(source, message, help, notes, location, line, column, changed)`. +type CompletedRow = ( + usize, + String, + Option, + Vec, + Option, + Option, + Option, + bool, +); +/// `(completed position, (model, message, note, help))`: exactly one side is set. +type OrderRow = (Option, Option<(usize, String, String, String)>); +/// `(deferral, completed, model binding positions, order)`. +type OutcomeRow = ( + Option<&'static str>, + Vec, + Option>>, + Vec, +); + +/// Recover, explain and reject opt-outs on `catalog`'s analysis pool. +#[pyfunction] +fn complete_semantic_checks( + py: Python<'_>, + catalog: PyRef<'_, ProjectCatalog>, + request: RequestInput, +) -> PyResult { + let request = completion_request(request); + let catalog = &catalog.inner; + let outcome = py + .compiler_detach(|| complete_semantic_diagnostics(&request, catalog)) + .map_err(compiler_error)?; + let (deferral, parts) = outcome.into_parts(); + let (diagnostics, model_bindings, order) = parts.unwrap_or_default(); + Ok(( + deferral, + diagnostics.into_iter().map(completed_row).collect(), + model_bindings, + order.into_iter().map(order_row).collect(), + )) +} + +fn location(row: LocationRow) -> SemanticLocation { + SemanticLocation { + line: row.0, + column: row.1, + end_line: row.2, + end_column: row.3, + } +} + +fn location_row(location: SemanticLocation) -> LocationRow { + ( + location.line, + location.column, + location.end_line, + location.end_column, + ) +} + +fn completion_request((dialect, diagnostics, models, shapes): RequestInput) -> CompletionRequest { + CompletionRequest { + dialect, + diagnostics: diagnostics.into_iter().map(semantic_diagnostic).collect(), + models: models.into_iter().map(completion_model).collect(), + shapes, + } +} + +fn semantic_diagnostic( + (id, code, message, resource_type, resource_name, line, column, place, help, notes): DiagnosticInput, +) -> SemanticDiagnostic { + SemanticDiagnostic { + id, + code, + message, + resource_type, + resource_name, + line, + column, + location: place.map(location), + help, + notes, + } +} + +fn completion_model( + (name, query_sql, authored_sql, inferred_columns, lineage, binding_codes, opt_out): ModelInput, +) -> CompletionModel { + CompletionModel { + name, + query_sql, + authored_sql, + inferred_columns, + lineage: lineage_outputs(lineage), + binding_codes, + rejected_opt_out_file: opt_out, + } +} + +fn completed_row(diagnostic: CompletedDiagnostic) -> CompletedRow { + ( + diagnostic.source, + diagnostic.message, + diagnostic.help, + diagnostic.notes, + diagnostic.location.map(location_row), + diagnostic.line, + diagnostic.column, + diagnostic.changed, + ) +} + +fn order_row(entry: FinalDiagnostic) -> OrderRow { + let (position, opt_out) = entry.into_parts(); + ( + position, + opt_out.map(|opt_out| (opt_out.model, opt_out.message, opt_out.note, opt_out.help)), + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(complete_semantic_checks, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs index 7444bd0374..03829ea183 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs @@ -1 +1,3 @@ +pub(crate) mod completion; pub(crate) mod registration; +pub(crate) mod type_recovery; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs index 7ee7c2c7f0..1ad08dbada 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs @@ -2,6 +2,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { - Ok(()) +use crate::bindings::_helpers::semantic_checks::{completion, type_recovery}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + type_recovery::register(module)?; + completion::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs new file mode 100644 index 0000000000..29d48a1739 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs @@ -0,0 +1,163 @@ +//! Native output-type recovery for failed models, with Python's revalidation in between. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyRef, PyResult, Python}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_analysis::semantic_checks::main::finish_type_recovery::finish_type_recovery; +use sqlbuild_analysis::semantic_checks::main::plan_type_recovery::plan_type_recovery; +use sqlbuild_analysis::semantic_checks::models::{ + DiagnosticOwner, LineageOutput, ModelBinding, RawBinding, RecoveryModel, TypeRecoveryPlan, + TypeRecoveryRequest, TypeRecoveryStep, +}; +use sqlbuild_analysis::semantic_checks::types::RevisedBinding; + +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::models::ProjectCatalog; +use crate::bindings::types::CompilerDetach; + +/// `(output_column, [(resource_name, column_name)])`. +pub(crate) type LineageInput = (String, Vec<(String, String)>); +/// `(name, query_sql, inferred names, reference names, lineage, bindings, raw bindings)`. +type ModelInput = ( + String, + String, + Option>, + Vec, + Vec, + Vec<(usize, String, String, bool)>, + Vec<(usize, String, String, Option, Option)>, +); +/// `(dialect, models, [(id, code, is_model, resource_name)])`. +type RequestInput = ( + Option, + Vec, + Vec<(usize, String, bool, Option)>, +); +/// `(kept [(diagnostic index, appended note)], model binding positions)`. +type OutcomeRow = (Vec<(usize, Option)>, Vec>); + +/// One planned type recovery, held between the native plan and Python's revalidation. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct SemanticTypeRecovery { + request: TypeRecoveryRequest, + step: TypeRecoveryStep, +} + +#[pymethods] +impl SemanticTypeRecovery { + /// `unchanged`, `planned` or `deferred`. + #[getter] + fn status(&self) -> &'static str { + self.step.status() + } + + /// The deferral kind, when the stage is handed back to Python. + #[getter] + fn deferral(&self) -> Option<&'static str> { + self.step.deferral() + } + + /// Poisoned `(model, column)` outputs in Python's dict order. + #[getter] + fn poisoned(&self) -> Vec<(String, String)> { + self.step + .plan() + .map(TypeRecoveryPlan::poisoned_outputs) + .unwrap_or_default() + } + + /// Model indexes Python revalidates with unknown poisoned types, in order. + #[getter] + fn revalidated(&self) -> Vec { + self.step + .plan() + .map(TypeRecoveryPlan::revalidated_models) + .unwrap_or_default() + } + + /// Retained diagnostics and model binding positions, or None to defer to Python. + fn finish(&self, revised: Vec>) -> Option { + let plan = self.step.plan()?; + match finish_type_recovery(&self.request, plan, &revised) { + Ok(outcome) => Some((outcome.kept, outcome.model_bindings)), + Err(_) => None, + } + } +} + +/// Plan type recovery on `catalog`'s analysis pool. +#[pyfunction] +fn plan_semantic_type_recovery( + py: Python<'_>, + catalog: PyRef<'_, ProjectCatalog>, + request: RequestInput, +) -> PyResult { + let request = recovery_request(request); + let catalog = &catalog.inner; + let step = py + .compiler_detach(|| plan_type_recovery(&request, catalog)) + .map_err(compiler_error)?; + Ok(SemanticTypeRecovery { request, step }) +} + +pub(crate) fn lineage_outputs(lineage: Vec) -> Vec { + lineage + .into_iter() + .map(|(output_column, upstream)| LineageOutput { + output_column, + upstream, + }) + .collect() +} + +fn recovery_request((dialect, models, diagnostics): RequestInput) -> TypeRecoveryRequest { + TypeRecoveryRequest { + dialect, + models: models.into_iter().map(recovery_model).collect(), + diagnostics: diagnostics + .into_iter() + .map(|(id, code, is_model, resource_name)| DiagnosticOwner { + id, + code, + is_model, + resource_name, + }) + .collect(), + } +} + +fn recovery_model( + (name, query_sql, inferred_columns, references, lineage, bindings, raw_bindings): ModelInput, +) -> RecoveryModel { + RecoveryModel { + name, + query_sql, + inferred_columns, + references, + lineage: lineage_outputs(lineage), + bindings: bindings + .into_iter() + .map(|(id, code, message, is_error)| ModelBinding { + id, + code, + message, + is_error, + }) + .collect(), + raw_bindings: raw_bindings + .into_iter() + .map(|(id, code, message, start, end)| RawBinding { + id, + code, + message, + start, + end, + }) + .collect(), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(plan_semantic_type_recovery, module)?)?; + Ok(()) +} diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py b/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py index 069b7d2986..62a3295c68 100644 --- a/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py +++ b/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py @@ -1,9 +1,95 @@ """Failure cases owned by the native semantic completion lane.""" +from scripts.compiler_differential._helpers.corpus.case_builder import ( + config_files, + failure_case, + mart_body_files, + staging_files, +) +from scripts.compiler_differential.constants import FAILURE_BASE_STAGING, FAILURE_MART_PATH from scripts.compiler_differential.models import FailureCase +_ORDERS: str = '__ref("stg_orders")' +_TIMED_STAGING: str = FAILURE_BASE_STAGING.replace( + "amount, status", "amount, status, CAST('2026-01-01' AS TIMESTAMP) AS ordered_at" +) + def semantic_checks_failure_cases() -> tuple[FailureCase, ...]: """Return this lane's failure cases; the lane appends here without editing shared lists.""" - return () + return ( + failure_case( + name="semantic-recovered-downstream-uses", + expected_code="B002", + expected_message="Unknown column 'amonut' in raw_orders", + expected_help="did you mean 'amount'?", + expected_notes=( + "raw_orders has: amount, status, customer_id, order_id", + "1 downstream uses of stg_orders.amount were not checked because of this error", + ), + files={ + **staging_files(FAILURE_BASE_STAGING.replace("amount, status", "amonut, status")), + **mart_body_files( + f"SELECT customer_id, SUM(amount) AS total_amount\nFROM {_ORDERS}\n" + "GROUP BY customer_id\n" + ), + }, + ), + failure_case( + name="semantic-qualified-alias-unknown-column", + expected_code="B002", + expected_message="Unknown column 'amonut' in stg_orders (as o)", + expected_help="did you mean 'amount'?", + files=mart_body_files( + f"SELECT o.customer_id, SUM(o.amonut) AS total_amount\nFROM {_ORDERS} AS o\n" + "GROUP BY o.customer_id\n" + ), + ), + failure_case( + name="semantic-temporal-comparison-help", + expected_code="B217", + expected_help="compare with a timestamp, for example TIMESTAMP '2026-04-01'", + expected_notes=("o.ordered_at is TIMESTAMP, 5 is INTEGER",), + expected_location=(7, 7), + files={ + **staging_files(_TIMED_STAGING), + **mart_body_files( + f"SELECT o.customer_id, SUM(o.amount) AS total_amount\nFROM {_ORDERS} AS o\n" + "WHERE o.ordered_at > 5\nGROUP BY o.customer_id\n" + ), + }, + ), + failure_case( + name="semantic-type-error-unchecked-uses", + expected_code="B212", + expected_notes=( + "1 downstream output uses were not type-checked because of this error", + "status is TEXT, 1 is INTEGER", + ), + files={ + **staging_files( + FAILURE_BASE_STAGING.replace("amount, status", "status + 1 AS amount, status") + ), + **mart_body_files( + f"SELECT customer_id, SUM(amount) AS total_amount\nFROM {_ORDERS}\n" + "GROUP BY customer_id\n" + ), + }, + ), + failure_case( + name="semantic-rejected-opt-out-counts-findings", + expected_code="P009", + expected_message="`sql_analysis false` is not needed for model 'customer_totals'", + expected_help="hiding: 2 unknown columns, 1 type mismatch (run `sqb compile`", + files={ + **config_files("\n[settings]\nrequire_sql_analysis = true\n"), + FAILURE_MART_PATH: ( + 'MODEL (\n description "Order totals per customer",\n' + " sql_analysis false,\n);\n\n" + "SELECT customer_id, SUM(refund) AS total_amount, MAX(amonut) AS top_amount\n" + f"FROM {_ORDERS}\nWHERE status > 5\nGROUP BY customer_id\n" + ), + }, + ), + ) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 75d48ddb6c..1cd39de451 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -411,6 +411,87 @@ def infer_expression_source_shapes( ) -> tuple[list[tuple[bool, list[tuple[str, str]] | None]], str | None]: ... # Native analysis: semantic completion. +class SemanticTypeRecovery: + @property + def status(self) -> str: ... + @property + def deferral(self) -> str | None: ... + @property + def poisoned(self) -> list[tuple[str, str]]: ... + @property + def revalidated(self) -> list[int]: ... + def finish( + self, revised: list[list[tuple[str, int | None, int | None]]], / + ) -> tuple[list[tuple[int, str | None]], list[list[int]]] | None: ... + +def plan_semantic_type_recovery( + catalog: object, + request: tuple[ + str | None, + list[ + tuple[ + str, + str, + list[str] | None, + list[str], + list[tuple[str, list[tuple[str, str]]]], + list[tuple[int, str, str, bool]], + list[tuple[int, str, str, int | None, int | None]], + ] + ], + list[tuple[int, str, bool, str | None]], + ], + /, +) -> SemanticTypeRecovery: ... +def complete_semantic_checks( + catalog: object, + request: tuple[ + str | None, + list[ + tuple[ + int, + str, + str, + str | None, + str | None, + int | None, + int | None, + tuple[int, int, int | None, int | None] | None, + str | None, + list[str], + ] + ], + list[ + tuple[ + str, + str, + str, + list[str], + list[tuple[str, list[tuple[str, str]]]], + list[str], + str | None, + ] + ], + list[tuple[str, list[tuple[str, str]]]], + ], + /, +) -> tuple[ + str | None, + list[ + tuple[ + int, + str, + str | None, + list[str], + tuple[int, int, int | None, int | None] | None, + int | None, + int | None, + bool, + ] + ], + list[list[int]] | None, + list[tuple[int | None, tuple[int, str, str, str] | None]], +]: ... # Native analysis: contracts. diff --git a/src/sqlbuild/compiler/compile/classes/semantic_completion_inputs.py b/src/sqlbuild/compiler/compile/classes/semantic_completion_inputs.py new file mode 100644 index 0000000000..a617ba2114 --- /dev/null +++ b/src/sqlbuild/compiler/compile/classes/semantic_completion_inputs.py @@ -0,0 +1,66 @@ +"""Compile-owned inputs of the native semantic completion stage.""" + +from __future__ import annotations + +from dataclasses import replace + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile._helpers.analysis.compact import get_complete_schema_binding_request +from sqlbuild.compiler.compile._helpers.assembly.metadata_validation import ( + get_semantic_metadata_diagnostics, +) +from sqlbuild.compiler.compile._helpers.assembly.native_declarations import ( + known_declared_types, + known_function_names, +) +from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes +from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.sql_analysis.main._identifier_case import ignores_quoted_case +from sqlbuild.compiler.sql_analysis.models import SqlSchemaValidationRequest + + +class SemanticCompletionInputs: + """Expose the shapes, revalidation requests and metadata checks semantic completion reads.""" + + @staticmethod + def shapes(project: CompiledProject) -> dict[str, dict[str, str]]: + """Closed relation shapes, as type recovery, recovery and explanation read them.""" + + return semantic_shapes(project=project) + + @staticmethod + def revalidation_request( + *, project: CompiledProject, model: CompiledModel, shapes: dict[str, dict[str, str]] + ) -> SqlSchemaValidationRequest: + """Type recovery's request revalidating one failed model against `shapes`.""" + + return replace( + get_complete_schema_binding_request( + query_sql=model.query_sql, + placeholders=None, + dialect=project.sql_analysis_dialect, + known_functions=known_function_names(project.functions), + known_types=known_declared_types(functions=project.functions, column_types=shapes), + binding_schema={ + reference.ref_name: shapes.get(reference.ref_name, {}) + for reference in model.references + }, + ), + catalog=project.binding_catalog, + quoted_identifiers_ignore_case=ignores_quoted_case( + connection=project.effective_connection, dialect=project.sql_analysis_dialect + ), + ) + + @staticmethod + def metadata_diagnostics( + *, + project: CompiledProject, + profile: ExpressionInferenceProfile, + resource_sql_analysis: bool, + ) -> tuple[CompilerDiagnostic, ...]: + """The semantic metadata checks after type recovery.""" + + return get_semantic_metadata_diagnostics( + project=project, profile=profile, resource_sql_analysis=resource_sql_analysis + ) diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/__init__.py b/src/sqlbuild/compiler/semantic_checks/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py b/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py new file mode 100644 index 0000000000..a54b887900 --- /dev/null +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py @@ -0,0 +1,153 @@ +"""Native diagnostic recovery, explanations and opt-out rejection after the metadata checks.""" + +from __future__ import annotations + +from dataclasses import replace +from typing import Any + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.classes.semantic_completion_inputs import SemanticCompletionInputs +from sqlbuild.compiler.compile.constants import UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE +from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.compile.types import ( + CompiledResourceType, + DiagnosticPhase, + DiagnosticSeverity, +) +from sqlbuild.compiler.semantic_checks._helpers.deferrals import record_semantic_deferral +from sqlbuild.compiler.semantic_checks._helpers.payloads import diagnostic_ids, lineage_payload +from sqlbuild.compiler.semantic_checks.constants import COMPLETION_DEFERRAL_SITE +from sqlbuild.spec.contracts.models import SourceLocation + +_RECOVERED_CODE: str = "B002" + +type _Location = tuple[int, int, int | None, int | None] +type _Completed = tuple[ + int, str, str | None, list[str], _Location | None, int | None, int | None, bool +] + + +def complete_native_diagnostics( + *, project: CompiledProject, catalog: object +) -> CompiledProject | None: + """Return the recovered, explained and opt-out-checked project, or None to use Python.""" + + ids: dict[CompilerDiagnostic, int] = diagnostic_ids(project) + recovers: bool = any(item.code == _RECOVERED_CODE for item in project.diagnostics) + shapes: dict[str, dict[str, str]] = ( + SemanticCompletionInputs.shapes(project) if project.diagnostics else {} + ) + request: Any = ( + project.sql_analysis_dialect, + [_diagnostic_payload(item=item, ids=ids) for item in project.diagnostics], + [_model_payload(model=model, lineage=recovers) for model in project.models], + [(name, list(columns.items())) for name, columns in shapes.items()], + ) + deferral, completed, model_bindings, order = _native.complete_semantic_checks(catalog, request) + if deferral is not None: + record_semantic_deferral(kind=deferral, site=COMPLETION_DEFERRAL_SITE) + return None + diagnostics: list[CompilerDiagnostic] = [ + _completed(original=project.diagnostics[row[0]], row=row) for row in completed + ] + models: tuple[CompiledModel, ...] = project.models + if model_bindings is not None: + models = tuple( + replace( + model, binding_diagnostics=_positions(diagnostics=diagnostics, positions=positions) + ) + for model, positions in zip(project.models, model_bindings, strict=True) + ) + final: list[CompilerDiagnostic] = [] + for position, opt_out in order: + if position is not None: + final.append(diagnostics[position]) + elif opt_out is not None: + index, message, note, help_text = opt_out + final.append( + _opt_out( + model=project.models[index], message=message, note=note, help_text=help_text + ) + ) + return replace(project, models=models, diagnostics=tuple(final)) + + +def _positions( + *, diagnostics: list[CompilerDiagnostic], positions: list[int] +) -> tuple[CompilerDiagnostic, ...]: + return tuple(diagnostics[position] for position in positions) + + +def _diagnostic_payload( + *, item: CompilerDiagnostic, ids: dict[CompilerDiagnostic, int] +) -> tuple[Any, ...]: + location: SourceLocation | None = item.location + return ( + ids[item], + item.code, + item.message, + None if item.resource_type is None else str(item.resource_type), + item.resource_name, + item.line, + item.column, + None + if location is None + else (location.line, location.column, location.end_line, location.end_column), + item.help, + list(item.notes), + ) + + +def _model_payload(*, model: CompiledModel, lineage: bool) -> tuple[Any, ...]: + opt_out: SourceLocation | None = model.rejected_sql_analysis_opt_out + return ( + model.name, + model.query_sql, + model.authored_sql, + [column.name for column in model.inferred_columns or ()], + lineage_payload(model) if lineage else [], + [item.code for item in model.binding_diagnostics], + None if opt_out is None else opt_out.path.name, + ) + + +def _completed(*, original: CompilerDiagnostic, row: _Completed) -> CompilerDiagnostic: + _, message, help_text, notes, location, line, column, changed = row + if not changed: + return original + current: SourceLocation | None = original.location + if current is not None and location is not None: + line_number, column_number, end_line, end_column = location + if (current.line, current.column, current.end_line, current.end_column) != location: + current = replace( + current, + line=line_number, + column=column_number, + end_line=end_line, + end_column=end_column, + ) + return replace( + original, + message=message, + help=help_text, + notes=tuple(notes), + location=current, + line=line, + column=column, + ) + + +def _opt_out( + *, model: CompiledModel, message: str, note: str, help_text: str +) -> CompilerDiagnostic: + return CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR, + code=UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE, + message=message, + resource_type=CompiledResourceType.MODEL, + resource_name=model.name, + location=model.rejected_sql_analysis_opt_out, + notes=(note,), + help=help_text, + ) diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/deferrals.py b/src/sqlbuild/compiler/semantic_checks/_helpers/deferrals.py new file mode 100644 index 0000000000..67a2ec2bea --- /dev/null +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/deferrals.py @@ -0,0 +1,23 @@ +"""Debug-only record of semantic completion stages the native engine hands back to Python.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +from sqlbuild.compiler.lineage.constants import ANALYSIS_DEFERRAL_RECORD_PREFIX +from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR + + +def record_semantic_deferral(*, kind: str, site: str) -> None: + """Append one `{"kind", "site"}` line when the analysis record directory is set.""" + + directory: str | None = os.environ.get(ANALYSIS_RECORD_DIR_ENV_VAR) + if not directory: + return + target: Path = Path(directory) + target.mkdir(parents=True, exist_ok=True) + record: str = json.dumps({"kind": kind, "site": site}) + with open(target / f"{ANALYSIS_DEFERRAL_RECORD_PREFIX}{os.getpid()}.jsonl", "a") as handle: + _ = handle.write(record + "\n") diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py b/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py new file mode 100644 index 0000000000..b05811fc97 --- /dev/null +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py @@ -0,0 +1,58 @@ +"""Plain-data payloads of compiled facts for the native semantic completion requests.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile.models import ( + CompiledLineageColumnFact, + CompiledModel, + CompiledProject, + CompilerDiagnostic, +) +from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic + + +def lineage_payload(model: CompiledModel) -> list[tuple[str, list[tuple[str, str]]]]: + """Each output column with the `(resource, column)` pairs it reads.""" + + return [_output_payload(output) for output in model.fast_lineage_columns or ()] + + +def _output_payload(output: CompiledLineageColumnFact) -> tuple[str, list[tuple[str, str]]]: + return ( + output.output_column, + [(source.resource_name, source.column_name) for source in output.upstream_columns], + ) + + +def diagnostic_ids(project: CompiledProject) -> dict[CompilerDiagnostic, int]: + """One id per distinct diagnostic value, as Python's value-keyed dicts and sets see them.""" + + ids: dict[CompilerDiagnostic, int] = {} + for item in project.diagnostics: + _ = ids.setdefault(item, len(ids)) + for model in project.models: + for item in model.binding_diagnostics: + _ = ids.setdefault(item, len(ids)) + return ids + + +def raw_binding_ids( + binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], +) -> dict[SqlBindingDiagnostic, int]: + """One id per distinct raw binding row value.""" + + ids: dict[SqlBindingDiagnostic, int] = {} + for rows in binding_results.values(): + for row in rows: + _ = ids.setdefault(row, len(ids)) + return ids + + +def has_blocking_binding(project: CompiledProject) -> bool: + """Whether any model owns a binding error, without which type recovery changes nothing.""" + + for model in project.models: + for item in model.binding_diagnostics: + if item.is_error: + return True + return False diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py b/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py new file mode 100644 index 0000000000..46e18968c8 --- /dev/null +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py @@ -0,0 +1,48 @@ +"""The native semantic completion stage: type recovery, metadata checks, then completion.""" + +from __future__ import annotations + +from dataclasses import replace + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile.classes.semantic_completion_inputs import SemanticCompletionInputs +from sqlbuild.compiler.compile.models import CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.semantic_checks._helpers.completion import complete_native_diagnostics +from sqlbuild.compiler.semantic_checks._helpers.deferrals import record_semantic_deferral +from sqlbuild.compiler.semantic_checks._helpers.type_recovery import recover_native_output_types +from sqlbuild.compiler.semantic_checks.constants import ( + COMPLETION_DEFERRAL_SITE, + METADATA_DEFERRAL_SITE, + NATIVE_SEMANTIC_METADATA_CHECKS, + NATIVE_SEMANTIC_NO_CATALOG, +) +from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic + + +def completed_semantic_project( + *, + project: CompiledProject, + profile: ExpressionInferenceProfile, + binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], + resource_sql_analysis: bool, +) -> CompiledProject | None: + """Python's completed project, or None where any native step hands the stage back.""" + + if project.binding_catalog is None: + record_semantic_deferral(kind=NATIVE_SEMANTIC_NO_CATALOG, site=COMPLETION_DEFERRAL_SITE) + return None + catalog: object = project.binding_catalog.native + recovered: CompiledProject | None = recover_native_output_types( + project=project, binding_results=binding_results, catalog=catalog + ) + if recovered is None: + return None + if recovered.settings.sql_analysis: + record_semantic_deferral(kind=NATIVE_SEMANTIC_METADATA_CHECKS, site=METADATA_DEFERRAL_SITE) + metadata: tuple[CompilerDiagnostic, ...] = SemanticCompletionInputs.metadata_diagnostics( + project=recovered, profile=profile, resource_sql_analysis=resource_sql_analysis + ) + return complete_native_diagnostics( + project=replace(recovered, diagnostics=(*recovered.diagnostics, *metadata)), + catalog=catalog, + ) diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py b/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py new file mode 100644 index 0000000000..c4ade28bdb --- /dev/null +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py @@ -0,0 +1,178 @@ +"""Native output-type recovery, with Python's revalidation on the compile's binding catalog.""" + +from __future__ import annotations + +from dataclasses import replace +from typing import Any + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.classes.semantic_completion_inputs import SemanticCompletionInputs +from sqlbuild.compiler.compile.models import ( + CompiledModel, + CompiledProject, + CompilerDiagnostic, + InferredColumn, +) +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.semantic_checks._helpers.deferrals import record_semantic_deferral +from sqlbuild.compiler.semantic_checks._helpers.payloads import ( + diagnostic_ids, + has_blocking_binding, + lineage_payload, + raw_binding_ids, +) +from sqlbuild.compiler.semantic_checks.constants import ( + NATIVE_SEMANTIC_FAILURE, + NATIVE_TYPE_RECOVERY_DEFERRED, + NATIVE_TYPE_RECOVERY_UNCHANGED, + TYPE_RECOVERY_DEFERRAL_SITE, + UNKNOWN_RECOVERED_TYPE, +) +from sqlbuild.compiler.sql_analysis.main._schema_validation import get_schema_validations +from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic, SqlBindingResult + +type _Revised = list[tuple[str, int | None, int | None]] + + +def recover_native_output_types( + *, + project: CompiledProject, + binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], + catalog: object, +) -> CompiledProject | None: + """Return `recover_output_types`'s project, or None where Python must recover the types.""" + + if not has_blocking_binding(project): + return project + ids: dict[CompilerDiagnostic, int] = diagnostic_ids(project) + raw_ids: dict[SqlBindingDiagnostic, int] = raw_binding_ids(binding_results) + request: Any = ( + project.sql_analysis_dialect, + [ + _model_payload(model=model, binding_results=binding_results, ids=ids, raw_ids=raw_ids) + for model in project.models + ], + [ + ( + ids[item], + item.code, + item.resource_type == CompiledResourceType.MODEL, + item.resource_name, + ) + for item in project.diagnostics + ], + ) + recovery: _native.SemanticTypeRecovery = _native.plan_semantic_type_recovery(catalog, request) + if recovery.status == NATIVE_TYPE_RECOVERY_UNCHANGED: + return project + if recovery.status == NATIVE_TYPE_RECOVERY_DEFERRED: + record_semantic_deferral(kind=str(recovery.deferral), site=TYPE_RECOVERY_DEFERRAL_SITE) + return None + poisoned: list[tuple[str, str]] = recovery.poisoned + outcome: tuple[list[tuple[int, str | None]], list[list[int]]] | None = recovery.finish( + _revalidated(project=project, revalidated=recovery.revalidated, poisoned=poisoned) + ) + if outcome is None: + record_semantic_deferral(kind=NATIVE_SEMANTIC_FAILURE, site=TYPE_RECOVERY_DEFERRAL_SITE) + return None + kept, model_bindings = outcome + diagnostics: tuple[CompilerDiagnostic, ...] = tuple( + _with_note(diagnostic=project.diagnostics[index], note=note) for index, note in kept + ) + poisoned_outputs: frozenset[tuple[str, str]] = frozenset(poisoned) + models: tuple[CompiledModel, ...] = tuple( + _recovered_model( + model=model, + bindings=_positions(diagnostics=diagnostics, positions=positions), + poisoned=poisoned_outputs, + ) + for model, positions in zip(project.models, model_bindings, strict=True) + ) + return replace(project, models=models, diagnostics=diagnostics) + + +def _revalidated( + *, project: CompiledProject, revalidated: list[int], poisoned: list[tuple[str, str]] +) -> list[_Revised]: + """Python's revalidation of each planned model with unknown types for poisoned outputs.""" + + if not revalidated: + return [] + shapes: dict[str, dict[str, str]] = _unknown_shapes( + shapes=SemanticCompletionInputs.shapes(project), poisoned=poisoned + ) + revised: list[_Revised] = [] + for index in revalidated: + result: SqlBindingResult = get_schema_validations( + requests=( + SemanticCompletionInputs.revalidation_request( + project=project, model=project.models[index], shapes=shapes + ), + ) + )[0] + revised.append([(item.code, item.start, item.end) for item in result.diagnostics]) + return revised + + +def _model_payload( + *, + model: CompiledModel, + binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], + ids: dict[CompilerDiagnostic, int], + raw_ids: dict[SqlBindingDiagnostic, int], +) -> tuple[Any, ...]: + raws: tuple[SqlBindingDiagnostic, ...] = ( + binding_results.get(model.name, ()) if model.binding_diagnostics else () + ) + return ( + model.name, + model.query_sql, + None + if model.inferred_columns is None + else [column.name for column in model.inferred_columns], + [reference.ref_name for reference in model.references], + lineage_payload(model), + [(ids[item], item.code, item.message, item.is_error) for item in model.binding_diagnostics], + [(raw_ids[raw], raw.code, raw.message, raw.start, raw.end) for raw in raws], + ) + + +def _with_note(*, diagnostic: CompilerDiagnostic, note: str | None) -> CompilerDiagnostic: + return replace(diagnostic, notes=(*diagnostic.notes, *((note,) if note is not None else ()))) + + +def _positions( + *, diagnostics: tuple[CompilerDiagnostic, ...], positions: list[int] +) -> tuple[CompilerDiagnostic, ...]: + return tuple(diagnostics[position] for position in positions) + + +def _recovered_model( + *, + model: CompiledModel, + bindings: tuple[CompilerDiagnostic, ...], + poisoned: frozenset[tuple[str, str]], +) -> CompiledModel: + columns: tuple[InferredColumn, ...] | None = None + if model.inferred_columns is not None: + columns = tuple( + replace(column, type=None) if (model.name, column.name) in poisoned else column + for column in model.inferred_columns + ) + unchecked: frozenset[str] = frozenset(column for name, column in poisoned if name == model.name) + return replace( + model, + binding_diagnostics=bindings, + inferred_columns=columns, + unchecked_output_columns=unchecked, + ) + + +def _unknown_shapes( + *, shapes: dict[str, dict[str, str]], poisoned: list[tuple[str, str]] +) -> dict[str, dict[str, str]]: + unknown: dict[str, dict[str, str]] = {name: dict(columns) for name, columns in shapes.items()} + for name, column in poisoned: + if name in unknown and column in unknown[name]: + unknown[name][column] = UNKNOWN_RECOVERED_TYPE + return unknown diff --git a/src/sqlbuild/compiler/semantic_checks/constants.py b/src/sqlbuild/compiler/semantic_checks/constants.py new file mode 100644 index 0000000000..6ea316eb6f --- /dev/null +++ b/src/sqlbuild/compiler/semantic_checks/constants.py @@ -0,0 +1,11 @@ +"""Native semantic completion statuses and deferral record sites.""" + +NATIVE_TYPE_RECOVERY_UNCHANGED: str = "unchanged" +NATIVE_TYPE_RECOVERY_DEFERRED: str = "deferred" +NATIVE_SEMANTIC_NO_CATALOG: str = "no_analysis_catalog" +NATIVE_SEMANTIC_METADATA_CHECKS: str = "metadata_checks" +NATIVE_SEMANTIC_FAILURE: str = "native_failure" +TYPE_RECOVERY_DEFERRAL_SITE: str = "type_recovery.py" +COMPLETION_DEFERRAL_SITE: str = "recovery.py" +METADATA_DEFERRAL_SITE: str = "metadata_validation.py" +UNKNOWN_RECOVERED_TYPE: str = "UNKNOWN" diff --git a/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py b/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py index ba3914b53f..762329e0a7 100644 --- a/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py +++ b/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py @@ -4,6 +4,7 @@ from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.semantic_checks._helpers.stage import completed_semantic_project from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic @@ -16,4 +17,9 @@ def complete_native_semantic_diagnostics( ) -> CompiledProject | None: """Return the completed project, or None where Python must complete its diagnostics.""" - return None + return completed_semantic_project( + project=project, + profile=profile, + binding_results=binding_results, + resource_sql_analysis=resource_sql_analysis, + ) diff --git a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py index e0dde5cd4c..f0184ff6c8 100644 --- a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py +++ b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py @@ -436,12 +436,14 @@ def test_given_failure_expectation_when_code_is_not_the_first_error_then_harness "test_case", [ WheelSiteReportTestCase( - description="python_and_preview_wheel_calls_with_input_enrichment_deferrals", + description="python_and_preview_wheel_calls_with_python_fallback_deferrals", perturbation="", expected_lines=( "Polyglot wheel calls (python):", "Polyglot wheel calls (native-preview):", "Analysis deferrals (python): none recorded", + "Analysis deferrals (native-preview):", + " metadata_checks metadata_validation.py (project ", " input_enrichment analysis_session (project ", "Compiler differential passed: 1 projects identical", ), @@ -451,7 +453,9 @@ def test_given_failure_expectation_when_code_is_not_the_first_error_then_harness "_validate_sql_syntax_with_message parse_one", } ), - expected_deferrals=frozenset({"input_enrichment analysis_session"}), + expected_deferrals=frozenset( + {"metadata_checks metadata_validation.py", "input_enrichment analysis_session"} + ), ), WheelSiteReportTestCase( description="preview_deferrals_are_counted_per_kind_and_site", @@ -463,7 +467,11 @@ def test_given_failure_expectation_when_code_is_not_the_first_error_then_harness ), expected_sites=frozenset(), expected_deferrals=frozenset( - {"legacy_fallback orders.sql", "input_enrichment analysis_session"} + { + "legacy_fallback orders.sql", + "metadata_checks metadata_validation.py", + "input_enrichment analysis_session", + } ), ), ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/__init__.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/_test_types.py new file mode 100644 index 0000000000..1f75e72cc4 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/_test_types.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class NativeSemanticChecksCliTestCase: + """Every engine compiling one failing project through the CLI with identical diagnostics.""" + + description: str + files: dict[str, str] + expected_codes: tuple[str, ...] + expected_notes: tuple[str, ...] + expected_preview_deferrals: tuple[tuple[str, str], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py new file mode 100644 index 0000000000..59b7847e73 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/helpers.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import json +import subprocess +from collections import Counter +from itertools import chain +from pathlib import Path +from typing import Any, NamedTuple, cast + +from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR +from sqlbuild.compiler.semantic_checks.constants import ( + COMPLETION_DEFERRAL_SITE, + METADATA_DEFERRAL_SITE, + TYPE_RECOVERY_DEFERRAL_SITE, +) +from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import run_installed_sqb + +SEMANTIC_WHEEL_SITES: tuple[str, ...] = ( + "compiler/compile/_helpers/diagnostics/type_recovery.py:_projection_spans", + "compiler/compile/_helpers/diagnostics/details.py:_parsed_model", +) +SEMANTIC_DEFERRAL_SITES: frozenset[str] = frozenset( + {COMPLETION_DEFERRAL_SITE, METADATA_DEFERRAL_SITE, TYPE_RECOVERY_DEFERRAL_SITE} +) + + +class EngineSemanticRun(NamedTuple): + """One engine's compile report, semantic wheel calls and semantic-stage deferrals.""" + + report: dict[str, object] + returncode: int + semantic_wheel_calls: int + deferrals: tuple[tuple[str, str], ...] + + +def engine_semantic_run( + *, project_dir: Path, files: dict[str, str], engine: str +) -> EngineSemanticRun: + """Write the project and compile it once under `engine` with analysis records on.""" + + for relative_path, contents in files.items(): + path: Path = project_dir / "project" / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + record_dir: Path = project_dir / "records" + compiled: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir / "project", + args=("--compiler-engine", engine, "compile", "--json", "--no-cache"), + env={REUSE_DISABLE_ENV_VAR: "1", ANALYSIS_RECORD_DIR_ENV_VAR: str(record_dir)}, + ) + report: dict[str, object] = json.loads(compiled.stdout.replace(str(project_dir), "")) + _ = report.pop("compiler_engine", None) + _ = report.pop("compile_timings", None) + calls: Counter[str] = Counter() + for path in record_dir.glob("polyglot-sites-*.json"): + for site, _, count in json.loads(path.read_text("utf-8"))["calls"]: + calls[site] += count + deferrals: list[tuple[str, str]] = [] + for path in sorted(record_dir.glob("analysis-deferrals-*.jsonl")): + for line in path.read_text("utf-8").splitlines(): + record: dict[str, str] = json.loads(line) + deferrals.append((record["kind"], record["site"])) + return EngineSemanticRun( + report=report, + returncode=compiled.returncode, + semantic_wheel_calls=sum(calls[site] for site in SEMANTIC_WHEEL_SITES), + deferrals=tuple(filter(lambda deferral: deferral[1] in SEMANTIC_DEFERRAL_SITES, deferrals)), + ) + + +def diagnostic_codes(report: dict[str, object]) -> tuple[str, ...]: + """Each reported diagnostic's code, in order.""" + + diagnostics: list[dict[str, Any]] = cast(list[dict[str, Any]], report["diagnostics"]) + return tuple(str(item["code"]) for item in diagnostics) + + +def diagnostic_notes(report: dict[str, object]) -> frozenset[str]: + """Every note line across the reported diagnostics.""" + + diagnostics: list[dict[str, Any]] = cast(list[dict[str, Any]], report["diagnostics"]) + return frozenset(chain.from_iterable(item.get("notes", ()) for item in diagnostics)) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/test_native_semantic_checks.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/test_native_semantic_checks.py new file mode 100644 index 0000000000..68d5eb9047 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/semantic_checks/test_native_semantic_checks.py @@ -0,0 +1,93 @@ +"""The preview engine completes semantic diagnostics natively with Python's exact CLI output.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile.semantic_checks._test_types import ( + NativeSemanticChecksCliTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.semantic_checks.helpers import ( + EngineSemanticRun, + diagnostic_codes, + diagnostic_notes, + engine_semantic_run, +) + +_ENGINES: tuple[str, ...] = ("python", "native", "native-preview") +_CONFIG: str = ( + 'name = "orders_semantics"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' +) +_SOURCES: str = ( + "sources:\n - name: raw_orders\n description: Orders feed.\n expression: >-\n" + " (SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount,\n" + " 'placed' AS status, TIMESTAMP '2026-01-01 00:00:00' AS ordered_at)\n" + " columns:\n - name: order_id\n type: INTEGER\n" + " - name: customer_id\n type: INTEGER\n - name: amount\n type: DOUBLE\n" + " - name: status\n type: VARCHAR\n - name: ordered_at\n type: TIMESTAMP\n" +) +_FAILING_FILES: dict[str, str] = { + "sqlbuild_project.toml": _CONFIG + "\n[settings]\nrequire_sql_analysis = true\n", + "sources/raw.yml": _SOURCES, + "models/staging/stg_orders.sql": ( + 'MODEL (description "Staged orders");\n\n' + "SELECT order_id, customer_id, amonut, status + 1 AS status_rank, ordered_at\n" + 'FROM __source("raw_orders")\n' + ), + "models/marts/customer_totals.sql": ( + 'MODEL (description "Totals per customer");\n\n' + "SELECT o.customer_id, SUM(o.amount) AS total_amount, MAX(o.status_rank) AS top_rank\n" + 'FROM __ref("stg_orders") AS o\nWHERE o.ordered_at > 5\nGROUP BY o.customer_id\n' + ), + "models/marts/customer_labels.sql": ( + 'MODEL (description "Labels per customer", sql_analysis false);\n\n' + "SELECT customer_id, UPPER(labell) AS label\n" + 'FROM __ref("stg_orders")\n' + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + NativeSemanticChecksCliTestCase( + description="a recovered typo, a poisoned type, a comparison and a rejected opt-out", + files=_FAILING_FILES, + expected_codes=("B217", "B002", "B212", "P009"), + expected_notes=( + "1 downstream uses of stg_orders.amount were not checked because of this error", + "1 downstream output uses were not type-checked because of this error", + "o.ordered_at is TIMESTAMP, 5 is INTEGER", + "raw_orders has: amount, status, ordered_at, customer_id, order_id", + ), + expected_preview_deferrals=(("metadata_checks", "metadata_validation.py"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_project_when_compiling_with_each_engine_then_diagnostics_match( + test_case: NativeSemanticChecksCliTestCase, tmp_path: Path +) -> None: + runs: dict[str, EngineSemanticRun] = { + engine: engine_semantic_run( + project_dir=tmp_path / engine, files=test_case.files, engine=engine + ) + for engine in _ENGINES + } + python: EngineSemanticRun = runs["python"] + preview: EngineSemanticRun = runs["native-preview"] + + assert python.returncode == 1 + assert diagnostic_codes(python.report) == test_case.expected_codes + assert set(test_case.expected_notes) <= diagnostic_notes(python.report) + assert python.semantic_wheel_calls > 0 + assert runs["native"] == python + assert (preview.report, preview.returncode) == (python.report, python.returncode) + assert preview.semantic_wheel_calls == 0 + assert preview.deferrals == test_case.expected_preview_deferrals + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index ea06219cb7..b42e71022d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -7,6 +7,7 @@ import pytest +from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.lineage.models import ProjectColumnLineage from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( NativeAnalysisSeamTestCase, @@ -32,7 +33,7 @@ "infer_native_expression_source_shapes": [ANY], "analyze_native_model_sql": [ANY], "assemble_native_sql_tests": [None], - "complete_native_semantic_diagnostics": [None], + "complete_native_semantic_diagnostics": [ANY], "evaluate_native_model_contracts": [None], "build_native_column_lineage": [ANY], "plan_native_sql_test_artifacts": [None], @@ -74,6 +75,7 @@ def test_given_project_when_compiling_with_each_engine_then_native_seams_answer_ assert isinstance(preview_seams["build_native_column_lineage"][0], ProjectColumnLineage) assert isinstance(preview_seams["analyze_native_model_sql"][0], dict) assert isinstance(preview_seams["infer_native_expression_source_shapes"][0], tuple) + assert isinstance(preview_seams["complete_native_semantic_diagnostics"][0], CompiledProject) assert (native_run.returncode, preview_run.returncode) == (0, 0) assert report_without_engine(native_run) == report_without_engine(python_run) assert report_without_engine(preview_run) == report_without_engine(python_run) diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/__init__.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/_test_types.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/_test_types.py new file mode 100644 index 0000000000..e3f9874dc0 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/_test_types.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class GeneratedSemanticParityTestCase: + """Seeded failing projects whose semantic completion Python and the native engine agree on.""" + + description: str + seed: int + count: int + model_count: int + dialects: tuple[str, ...] + expected_minimum_native_completions: int + expected_minimum_native_diagnostics: int + expected_minimum_type_recovery_plans: int + expected_minimum_codes: dict[str, int] + + +@dataclass(frozen=True) +class DeferredSemanticTestCase: + """A project the native stage hands back to Python, which records the deferral.""" + + description: str + dialect: str + keeps_catalog: bool + non_ascii_comment: bool + expected_kinds: tuple[tuple[str, str], ...] diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py new file mode 100644 index 0000000000..112162a298 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -0,0 +1,299 @@ +from __future__ import annotations + +import json +import random +from collections import Counter +from collections.abc import Callable +from dataclasses import dataclass, replace +from itertools import chain +from pathlib import Path +from typing import Any + +import pytest + +import sqlbuild._native as native_module +import sqlbuild.compiler.compile._helpers.assembly.project as assembly_project +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile._helpers.diagnostics.recovery import ( + complete_semantic_diagnostics, +) +from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.pipeline.main.graph import build_project_graph +from sqlbuild.compiler.semantic_checks.main._complete_native_semantic_diagnostics import ( + complete_native_semantic_diagnostics, +) +from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic + +_RELATIONS: dict[str, tuple[str, ...]] = { + '__source("raw_orders")': ( + "order_id", + "customer_id", + "amount", + "status", + "ordered_at", + "order_date", + ), +} +_SOURCES: str = """sources: + - name: raw_orders + description: Orders feed. + expression: >- + (SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount, 'placed' AS status, + TIMESTAMP '2026-01-01 00:00:00' AS ordered_at, DATE '2026-01-01' AS order_date) + columns: + - name: order_id + type: INTEGER + - name: customer_id + type: INTEGER + - name: amount + type: DOUBLE + - name: status + type: VARCHAR + - name: ordered_at + type: TIMESTAMP + - name: order_date + type: DATE +""" +_OPT_OUT_SHARE: float = 0.2 +_TYPO_SHARE: float = 0.45 +_MIN_TYPO_LENGTH: int = 4 +_ALIASES: tuple[tuple[str, str], ...] = (("", ""), ("o.", " AS o")) +_SETTINGS: tuple[str, str] = ("", "\n[settings]\nrequire_sql_analysis = true\n") +_OPT_OUTS: tuple[str, str] = ("", ", sql_analysis false") +_COMPLETION_STATUSES: tuple[str, str] = ("completion_native", "completion_deferred") +_SOURCE_COMPARISON_SHARE: float = 0.6 + +type _Template = Callable[[random.Random, str, tuple[str, ...]], tuple[str, tuple[str, ...]]] + + +@dataclass(frozen=True) +class SemanticInputs: + """The arguments the Python compile passed to `complete_semantic_diagnostics`.""" + + project: CompiledProject + profile: ExpressionInferenceProfile + binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]] + resource_sql_analysis: bool + + +def _typo(rng: random.Random, column: str) -> str: + index: int = rng.randrange(1, max(len(column) - 2, 2)) + swapped: str = ( + column[:index] + column[index + 1 : index + 2] + column[index] + column[index + 2 :] + ) + return (column, swapped)[len(column) >= _MIN_TYPO_LENGTH and rng.random() < _TYPO_SHARE] + + +def _projection( + rng: random.Random, relation: str, columns: tuple[str, ...] +) -> tuple[str, tuple[str, ...]]: + picked: list[str] = rng.sample(list(columns), k=min(len(columns), rng.randint(2, 4))) + written: list[str] = [_typo(rng, column) for column in picked] + prefix, suffix = rng.choice(_ALIASES) + sql: str = f"SELECT {', '.join(prefix + name for name in written)}\nFROM {relation}{suffix}" + intended: tuple[str, ...] = tuple( + rng.choice((name, meant)) for name, meant in zip(written, picked, strict=True) + ) + return sql, intended + + +def _comparison( + rng: random.Random, relation: str, columns: tuple[str, ...] +) -> tuple[str, tuple[str, ...]]: + relation, columns = ((relation, columns), next(iter(_RELATIONS.items())))[ + rng.random() < _SOURCE_COMPARISON_SHARE + ] + column: str = rng.choice(columns) + literal: str = rng.choice(("5", "'placed'", "DATE '2026-01-01'", "TIMESTAMP '2026-01-01'")) + kept: tuple[str, ...] = tuple(dict.fromkeys((columns[0], column))) + sql: str = ( + f"SELECT {', '.join('o.' + name for name in kept)}\nFROM {relation} AS o\n" + f"WHERE o.{_typo(rng, column)} > {literal}" + ) + return sql, kept + + +def _arithmetic( + rng: random.Random, relation: str, columns: tuple[str, ...] +) -> tuple[str, tuple[str, ...]]: + left, right = rng.sample(list(columns), k=2) + sql: str = f"SELECT {columns[0]}, {left} + {right} AS {left}\nFROM {relation}" + return sql, tuple(dict.fromkeys((columns[0], left))) + + +def _aggregate( + rng: random.Random, relation: str, columns: tuple[str, ...] +) -> tuple[str, tuple[str, ...]]: + key, value = rng.sample(list(columns), k=2) + sql: str = ( + f"WITH base AS (\n SELECT {key}, {_typo(rng, value)} FROM {relation}\n)\n" + f"SELECT {key}, MAX({value}) AS {value}\nFROM base\nGROUP BY 1" + ) + return sql, (key, value) + + +_TEMPLATES: tuple[_Template, ...] = (_projection, _comparison, _arithmetic, _aggregate) + + +def _has_two_columns(relation: tuple[str, tuple[str, ...]]) -> bool: + return len(relation[1]) > 1 + + +def generated_semantic_files( + *, rng: random.Random, model_count: int, require_analysis: bool +) -> dict[str, str]: + """A project whose models chain through typos, temporal comparisons and type errors.""" + + settings: str = _SETTINGS[require_analysis] + files: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders_semantics"\nadapter = "duckdb"\n\n[connection]\n' + f'database = "orders.duckdb"\n{settings}' + ), + "sources/raw.yml": _SOURCES, + } + relations: list[tuple[str, tuple[str, ...]]] = list(_RELATIONS.items()) + for index in range(model_count): + relation, columns = rng.choice(list(filter(_has_two_columns, relations))) + sql, outputs = _TEMPLATES[index % len(_TEMPLATES)](rng, relation, columns) + name: str = f"orders_model_{index}" + opt_out: str = _OPT_OUTS[rng.random() < _OPT_OUT_SHARE] + files[f"models/layer_{index % 3}/{name}.sql"] = ( + f'MODEL (description "Generated orders model {index}"{opt_out});\n\n{sql}\n' + ) + relations.append((f'__ref("{name}")', outputs)) + return files + + +def captured_semantic_inputs( + *, project_dir: Path, files: dict[str, str], monkeypatch: pytest.MonkeyPatch +) -> SemanticInputs: + """Compile with the Python engine and keep the semantic completion stage's inputs.""" + + for relative_path, contents in files.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + captured: list[SemanticInputs] = [] + + def capture(**arguments: Any) -> CompiledProject: + captured.append(SemanticInputs(**arguments)) + return complete_semantic_diagnostics(**arguments) + + with monkeypatch.context() as patch: + patch.setenv(COMPILER_ENGINE_ENV_VAR, "python") + patch.setattr(assembly_project, "complete_semantic_diagnostics", capture) + _ = build_project_graph( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + ) + return captured[0] + + +def with_dialect(inputs: SemanticInputs, dialect: str) -> SemanticInputs: + """The same stage inputs read under another SQL dialect.""" + + return replace( + inputs, + project=replace(inputs.project, sql_analysis_dialect=dialect), + profile=replace(inputs.profile, sql_analysis_dialect=dialect), + ) + + +def python_completion( + *, inputs: SemanticInputs, monkeypatch: pytest.MonkeyPatch +) -> CompiledProject: + """The Python engine's completed project.""" + + with monkeypatch.context() as patch: + patch.setenv(COMPILER_ENGINE_ENV_VAR, "python") + return complete_semantic_diagnostics( + project=inputs.project, + profile=inputs.profile, + binding_results=inputs.binding_results, + resource_sql_analysis=inputs.resource_sql_analysis, + ) + + +def completed_natively(inputs: SemanticInputs) -> CompiledProject: + """The native stage's completed project, which must not defer.""" + + project: CompiledProject | None = native_completion(inputs) + assert project is not None + return project + + +def note_kinds(project: CompiledProject, kinds: tuple[str, ...]) -> Counter[str]: + """How many notes of each kind the project's diagnostics carry.""" + + notes: list[str] = list(chain.from_iterable(item.notes for item in project.diagnostics)) + counts: Counter[str] = Counter() + for kind in kinds: + counts[kind] = sum(kind in note for note in notes) + return counts + + +def native_completion(inputs: SemanticInputs) -> CompiledProject | None: + """The native stage's completed project, or None when it defers.""" + + return complete_native_semantic_diagnostics( + project=inputs.project, + profile=inputs.profile, + binding_results=inputs.binding_results, + resource_sql_analysis=inputs.resource_sql_analysis, + ) + + +def completion_view(project: CompiledProject) -> tuple[object, ...]: + """Everything the stage decides: diagnostics and each model's binding and type facts.""" + + return ( + project.diagnostics, + tuple(_model_view(model) for model in project.models), + ) + + +def _model_view(model: CompiledModel) -> tuple[object, ...]: + return ( + model.name, + model.binding_diagnostics, + model.inferred_columns, + model.unchecked_output_columns, + ) + + +def record_native_statuses(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: + """Count native type recovery plans and completed or deferred completions.""" + + statuses: Counter[str] = Counter() + plan: Callable[..., Any] = native_module.plan_semantic_type_recovery + complete: Callable[..., Any] = native_module.complete_semantic_checks + + def counted_plan(*arguments: Any) -> Any: + recovery: Any = plan(*arguments) + statuses[f"type_recovery_{recovery.status}"] += 1 + return recovery + + def counted_complete(*arguments: Any) -> Any: + outcome: Any = complete(*arguments) + statuses[_COMPLETION_STATUSES[outcome[0] is not None]] += 1 + statuses["diagnostics_native"] += len(outcome[1]) + return outcome + + monkeypatch.setattr(native_module, "plan_semantic_type_recovery", counted_plan) + monkeypatch.setattr(native_module, "complete_semantic_checks", counted_complete) + return statuses + + +def deferral_records(record_dir: Path) -> tuple[tuple[str, str], ...]: + """Every `(kind, site)` deferral the native stage recorded below `record_dir`.""" + + records: list[tuple[str, str]] = [] + for path in sorted(record_dir.glob("analysis-deferrals-*.jsonl")): + for line in path.read_text("utf-8").splitlines(): + record: dict[str, str] = json.loads(line) + records.append((record["kind"], record["site"])) + return tuple(records) diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py new file mode 100644 index 0000000000..1aa6c1ab1d --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py @@ -0,0 +1,161 @@ +"""Native semantic completion equals Python's on generated failing projects.""" + +from __future__ import annotations + +import random +from collections import Counter +from dataclasses import replace +from pathlib import Path + +import pytest + +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR +from tests.integration.src.sqlbuild.compiler.semantic_checks._test_types import ( + DeferredSemanticTestCase, + GeneratedSemanticParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.semantic_checks.helpers import ( + SemanticInputs, + captured_semantic_inputs, + completed_natively, + completion_view, + deferral_records, + generated_semantic_files, + native_completion, + note_kinds, + python_completion, + record_native_statuses, + with_dialect, +) + +_NOTE_KINDS: tuple[str, ...] = ("downstream output uses", "downstream uses of", " has: ") + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedSemanticParityTestCase( + description="typos, aliases, temporal comparisons, type errors and opt-outs", + seed=20261009, + count=8, + model_count=12, + dialects=("duckdb", "postgres", "snowflake", "bigquery"), + expected_minimum_native_completions=32, + expected_minimum_native_diagnostics=200, + expected_minimum_type_recovery_plans=16, + expected_minimum_codes={ + "B002": 100, + "B212": 4, + "B217": 8, + "P009": 16, + "downstream output uses": 16, + "downstream uses of": 8, + " has: ": 40, + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_generated_failing_projects_when_completing_natively_then_matches_python( + test_case: GeneratedSemanticParityTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + rng: random.Random = random.Random(test_case.seed) + statuses: Counter[str] = record_native_statuses(monkeypatch=monkeypatch) + codes: Counter[str] = Counter() + expected_views: list[tuple[object, ...]] = [] + native_views: list[tuple[object, ...]] = [] + for index in range(test_case.count): + captured: SemanticInputs = captured_semantic_inputs( + project_dir=tmp_path / f"project_{index}", + files=generated_semantic_files( + rng=rng, model_count=test_case.model_count, require_analysis=index % 2 == 0 + ), + monkeypatch=monkeypatch, + ) + for dialect in test_case.dialects: + inputs: SemanticInputs = with_dialect(captured, dialect) + python: CompiledProject = python_completion(inputs=inputs, monkeypatch=monkeypatch) + codes.update(item.code for item in python.diagnostics) + codes.update(note_kinds(python, _NOTE_KINDS)) + expected_views.append(completion_view(python)) + native_views.append(completion_view(completed_natively(inputs))) + + assert native_views == expected_views + assert statuses["completion_deferred"] == 0 + assert statuses["type_recovery_deferred"] == 0 + assert statuses["completion_native"] >= test_case.expected_minimum_native_completions + assert statuses["diagnostics_native"] >= test_case.expected_minimum_native_diagnostics + assert statuses["type_recovery_planned"] >= test_case.expected_minimum_type_recovery_plans + assert { + code: min(codes[code], minimum) + for code, minimum in test_case.expected_minimum_codes.items() + } == test_case.expected_minimum_codes + + +@pytest.mark.parametrize( + "test_case", + [ + DeferredSemanticTestCase( + description="a dialect outside the native parser build", + dialect="mysql", + keeps_catalog=True, + non_ascii_comment=False, + expected_kinds=(("unsupported_dialect", "type_recovery.py"),), + ), + DeferredSemanticTestCase( + description="non-ASCII authored SQL in an explained model", + dialect="duckdb", + keeps_catalog=True, + non_ascii_comment=True, + expected_kinds=( + ("metadata_checks", "metadata_validation.py"), + ("non_ascii_text", "recovery.py"), + ), + ), + DeferredSemanticTestCase( + description="a project without an analysis catalog to run on", + dialect="duckdb", + keeps_catalog=False, + non_ascii_comment=False, + expected_kinds=(("no_analysis_catalog", "recovery.py"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_deferred_projects_when_completing_then_python_completes_and_records( + test_case: DeferredSemanticTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + record_dir: Path = tmp_path / "records" + captured: SemanticInputs = captured_semantic_inputs( + project_dir=tmp_path / "project", + files=generated_semantic_files(rng=random.Random(5), model_count=8, require_analysis=False), + monkeypatch=monkeypatch, + ) + inputs: SemanticInputs = with_dialect(captured, test_case.dialect) + project: CompiledProject = replace( + inputs.project, + binding_catalog=(None, inputs.project.binding_catalog)[test_case.keeps_catalog], + models=tuple( + replace( + model, + authored_sql=model.authored_sql + + ("", "-- caf\u00e9\n")[test_case.non_ascii_comment], + ) + for model in inputs.project.models + ), + ) + monkeypatch.setenv(ANALYSIS_RECORD_DIR_ENV_VAR, str(record_dir)) + + native: CompiledProject | None = native_completion(replace(inputs, project=project)) + + assert native is None + assert deferral_records(record_dir) == test_case.expected_kinds + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"])