feat(analysis): give native analysis routes parse trees, types and findings

Cargo.toml

@@ -176,6 +176,8 @@ sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" }
 sqlbuild-state = { path = "crates/sqlbuild-state" }
 # SQL lint Rules, quality Rules and the formatter.
 sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "cf79656d247277a30bc81ef39b6dee420e721221" }
+# Scopes, name resolution, types and lineage for the native analysis route.
+sqlquack-semantics = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" }
 # The lossless SQL tokenizer behind query fingerprints.
 sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" }
 sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "19c51fa29ab01dbef9dddd18e8bfc406ed8e65f5" }

crates/sqlbuild-analysis/Cargo.toml

@@ -26,6 +26,7 @@ sqlbuild-render.workspace = true
 sqlbuild-scopes.workspace = true
 sqlbuild-sqltext.workspace = true
 sqlclean.workspace = true
+sqlquack-semantics.workspace = true
 sqlquack-syntax.workspace = true
 sqlquack-syntax-postgres.workspace = true
 

crates/sqlbuild-analysis/src/analysis_route/_helpers/clickhouse_types.rs

@@ -0,0 +1,53 @@
+use crate::analysis_route::constants::{LOW_CARDINALITY_WRAPPER, NULLABLE_WRAPPER};
+
+/// `type_sql` without whitespace outside its quoted strings.
+pub(crate) fn unspaced(type_sql: &str) -> String {
+    let mut quoted: bool = false;
+    type_sql
+        .chars()
+        .filter(|character| {
+            if *character == '\'' {
+                quoted = !quoted;
+            }
+            quoted || !character.is_whitespace()
+        })
+        .collect()
+}
+
+/// The type inside its top-level `Nullable` and `LowCardinality` wrappers, and whether it is nullable.
+pub(crate) fn unwrapped(type_sql: &str) -> (&str, bool) {
+    let mut current: &str = type_sql.trim();
+    let mut nullable: bool = false;
+    loop {
+        if let Some(inner) = wrapped(current, NULLABLE_WRAPPER) {
+            nullable = true;
+            current = inner;
+        } else if let Some(inner) = wrapped(current, LOW_CARDINALITY_WRAPPER) {
+            current = inner;
+        } else {
+            return (current, nullable);
+        }
+    }
+}
+
+/// The argument of `wrapper(...)` when the whole of `type_sql` is that one call.
+fn wrapped<'a>(type_sql: &'a str, wrapper: &str) -> Option<&'a str> {
+    let head: &str = type_sql.get(..wrapper.len())?;
+    if !head.eq_ignore_ascii_case(wrapper) {
+        return None;
+    }
+    let inner: &str = type_sql[wrapper.len()..]
+        .trim_start()
+        .strip_prefix('(')?
+        .strip_suffix(')')?;
+    let mut depth: usize = 0;
+    for character in inner.chars() {
+        match character {
+            '(' => depth += 1,
+            ')' if depth == 0 => return None,
+            ')' => depth -= 1,
+            _ => {}
+        }
+    }
+    Some(inner.trim())
+}

crates/sqlbuild-analysis/src/analysis_route/_helpers/located_gap.rs

@@ -0,0 +1,70 @@
+//! Where a native parser's gap is, and why the parser cannot read the SQL there.
+
+use crate::analysis_route::models::{GapPosition, GapSite, LocatedGap, ParserGap};
+use crate::compiled_project::main::text_position::text_position;
+use crate::compiled_project::models::TextPosition;
+
+/// The gap at its line and column in `sql`, and its offset in `source` when the text holds.
+pub(crate) fn located(sql: &str, gap: ParserGap, source: Option<&str>) -> LocatedGap {
+    let before: &str = sql.get(..gap.offset).unwrap_or(sql);
+    let offset: usize = before.chars().count();
+    let position: TextPosition = text_position(sql, offset);
+    LocatedGap {
+        construct: gap.construct,
+        message: gap.message,
+        line: position.line,
+        column: position.column,
+        syntax: gap.syntax,
+        source_offset: source.and_then(|source| aligned(sql, offset, source)),
+    }
+}
+
+/// The character of `source` that `derived`'s `offset` came from, when the text before or after holds.
+pub(crate) fn aligned(derived: &str, offset: usize, source: &str) -> Option<usize> {
+    let derived: Vec<char> = derived.chars().collect();
+    let source: Vec<char> = source.chars().collect();
+    if offset > derived.len() {
+        return None;
+    }
+    if source.starts_with(&derived[..offset]) {
+        return Some(offset);
+    }
+    source
+        .ends_with(&derived[offset..])
+        .then(|| source.len() - (derived.len() - offset))
+}
+
+/// Why the dialect's native parser cannot read SQL, as P026 words it.
+pub(crate) fn unreadable_reason(dialect: &str, gap: &LocatedGap) -> String {
+    unreadable_at(dialect, gap, (gap.line, gap.column, ""))
+}
+
+/// P026's text for a gap at its site: the hook and file, then the parser's reason there.
+pub(crate) fn sited_reason(dialect: &str, gap: &LocatedGap, site: &GapSite<'_>) -> String {
+    let label: String = site
+        .label
+        .map_or_else(String::new, |label| format!(" {label}"));
+    let at: (usize, usize, &str) = match site.position {
+        GapPosition::File(position) => (position.line, position.column, ""),
+        GapPosition::Statement(position) => (position.line, position.column, " of its SQL"),
+        GapPosition::Analysed => (gap.line, gap.column, " of the analysed SQL"),
+    };
+    format!(
+        "{label} ({}): {}",
+        site.path,
+        unreadable_at(dialect, gap, at)
+    )
+}
+
+fn unreadable_at(dialect: &str, gap: &LocatedGap, at: (usize, usize, &str)) -> String {
+    let (line, column, within) = at;
+    let detail: String = if gap.message.is_empty() || gap.message == gap.construct {
+        String::new()
+    } else {
+        format!(": {}", gap.message)
+    };
+    format!(
+        "the {dialect} parser cannot read {} at line {line}, column {column}{within}{detail}",
+        gap.construct
+    )
+}

crates/sqlbuild-analysis/src/analysis_route/_helpers/marker_names.rs

@@ -0,0 +1,50 @@
+//! One analysis name per distinct relation marker call, so repeated calls read as one relation.
+
+use std::sync::LazyLock;
+
+use regex::{Captures, Regex};
+
+use crate::relation_markers::constants::MARKER_NAME_PREFIX;
+use crate::relation_markers::types::MarkerCall;
+
+static MARKER_NAME: LazyLock<Result<Regex, String>> = LazyLock::new(|| {
+    Regex::new(&format!(r"\b{MARKER_NAME_PREFIX}(\d+)\b")).map_err(|error| error.to_string())
+});
+
+/// `named` with each marker name renamed to its call's first index, padded to keep positions.
+pub(crate) fn first_call_names(named: &str, calls: &[MarkerCall]) -> Result<String, String> {
+    let pattern: &Regex = MARKER_NAME.as_ref().map_err(Clone::clone)?;
+    Ok(pattern
+        .replace_all(named, |captures: &Captures<'_>| {
+            let written: &str = captures.get(0).map_or("", |found| found.as_str());
+            let first: Option<usize> = match captures.get(1).map(|index| index.as_str().parse()) {
+                Some(Ok(index)) => first_index(calls, index),
+                Some(Err(_)) | None => None,
+            };
+            match first {
+                Some(first) => {
+                    let name: String = format!("{MARKER_NAME_PREFIX}{first}");
+                    format!("{name:<width$}", width = written.len())
+                }
+                None => written.to_owned(),
+            }
+        })
+        .into_owned())
+}
+
+/// The calls whose index is their call's first, in order, with that index.
+pub(crate) fn distinct_calls(calls: &[MarkerCall]) -> Vec<(usize, &MarkerCall)> {
+    let mut distinct: Vec<(usize, &MarkerCall)> = Vec::with_capacity(calls.len());
+    for (index, call) in calls.iter().enumerate() {
+        if first_index(calls, index) == Some(index) {
+            distinct.push((index, call));
+        }
+    }
+    distinct
+}
+
+/// The first index of the call at `index`, which is `index` unless an earlier call is the same.
+fn first_index(calls: &[MarkerCall], index: usize) -> Option<usize> {
+    let call: &MarkerCall = calls.get(index)?;
+    calls.iter().position(|earlier| earlier == call)
+}

crates/sqlbuild-analysis/src/analysis_route/_helpers/mod.rs

@@ -0,0 +1,6 @@
+pub(crate) mod clickhouse_types;
+pub(crate) mod located_gap;
+pub(crate) mod marker_names;
+pub(crate) mod native_analysis;
+pub(crate) mod revalidation;
+pub(crate) mod whole_gap;