feat(native): report declaration and scope errors natively

crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs

@@ -3,28 +3,41 @@
 use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python};
 use pyo3::{pyfunction, wrap_pyfunction};
 use sqlbuild_sqltext::compiler::main::declaration_references::scan_declaration_references;
-use sqlbuild_sqltext::compiler::models::{DeclarationReference, DeclarationReferenceKind};
+use sqlbuild_sqltext::compiler::models::{
+    DeclarationReference, DeclarationReferenceKind, DeclarationReferenceScan,
+    DeclarationReferenceStop,
+};
 
 use crate::bindings::_helpers::boundary::panics::compiler_error;
 use crate::bindings::types::CompilerDetach;
 
 /// `(kind code, name, member, start, end)`: kind 0 is an enum member, 1 a constant.
 type ReferenceRow = (u8, String, Option<String>, usize, usize);
+/// References and stop code: unclosed quote 0, block comment 1, invalid enum 2, constant 3.
+type ScanRow = (Vec<ReferenceRow>, Option<u8>);
 
-/// Return each SQL string's references in order, or `None` where Python must expand it.
+/// Return each SQL string's references and stopping error, or `None` where Python must expand it.
 #[pyfunction]
 fn scan_sql_declaration_references(
     py: Python<'_>,
     sqls: Vec<String>,
-) -> PyResult<Vec<Option<Vec<ReferenceRow>>>> {
-    let scanned: Vec<Option<Vec<DeclarationReference>>> = py
+) -> PyResult<Vec<Option<ScanRow>>> {
+    let scanned: Vec<Option<DeclarationReferenceScan>> = py
         .compiler_detach(|| Ok(scan_declaration_references(&sqls)))
         .map_err(compiler_error)?;
-    Ok(scanned.into_iter().map(reference_rows).collect())
+    Ok(scanned.into_iter().map(|scan| scan.map(scan_row)).collect())
 }
 
-fn reference_rows(references: Option<Vec<DeclarationReference>>) -> Option<Vec<ReferenceRow>> {
-    Some(references?.into_iter().map(reference_row).collect())
+fn scan_row(scan: DeclarationReferenceScan) -> ScanRow {
+    (
+        scan.references.into_iter().map(reference_row).collect(),
+        scan.stop.map(|stop| match stop {
+            DeclarationReferenceStop::UnclosedQuote => 0,
+            DeclarationReferenceStop::UnclosedBlockComment => 1,
+            DeclarationReferenceStop::Malformed(DeclarationReferenceKind::Enum) => 2,
+            DeclarationReferenceStop::Malformed(DeclarationReferenceKind::Constant) => 3,
+        }),
+    )
 }
 
 fn reference_row(reference: DeclarationReference) -> ReferenceRow {

crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/reference_syntax.rs

@@ -1,5 +1,7 @@
 //! Python's `@enum("name").MEMBER` and `@const("name")` patterns at one reference start.
 
+use sqlbuild_core::text::main::is_python_space::is_python_space;
+
 use crate::compiler::models::DeclarationReferenceKind;
 
 /// One matched reference: kind, name, member and the byte offset after it.
@@ -10,12 +12,14 @@ pub(crate) struct MatchedReference<'sql> {
     pub(crate) end: usize,
 }
 
-/// Why a reference start cannot be matched natively.
+/// What Python's patterns find at one `@`.
 pub(crate) enum ReferenceSyntax<'sql> {
     /// `@enum`/`@const` followed by a word character: not a reference, keep scanning.
     NotReference,
     Matched(MatchedReference<'sql>),
-    /// Malformed or Unicode-dependent text: Python raises or decides.
+    /// A reference start whose full pattern does not match: Python raises an invalid reference.
+    Malformed(DeclarationReferenceKind),
+    /// A non-ASCII character after the keyword, whose word boundary only Python decides.
     Deferred,
 }
 
@@ -30,14 +34,20 @@ pub(crate) fn match_reference(sql: &str, start: usize) -> ReferenceSyntax<'_> {
         return ReferenceSyntax::NotReference;
     };
     match bytes.get(keyword_end) {
-        Some(byte) if !byte.is_ascii() => return ReferenceSyntax::Deferred,
+        Some(byte)
+            if !byte.is_ascii()
+                && !sql[keyword_end..]
+                    .chars()
+                    .next()
+                    .is_some_and(is_python_space) =>
+        {
+            return ReferenceSyntax::Deferred;
+        }
         Some(byte) if is_word(*byte) => return ReferenceSyntax::NotReference,
         _ => {}
     }
-    let Some(matched) = match_arguments(sql, kind, keyword_end) else {
-        return ReferenceSyntax::Deferred;
-    };
-    ReferenceSyntax::Matched(matched)
+    match_arguments(sql, kind, keyword_end)
+        .map_or(ReferenceSyntax::Malformed(kind), ReferenceSyntax::Matched)
 }
 
 fn match_arguments(
@@ -46,19 +56,19 @@ fn match_arguments(
     keyword_end: usize,
 ) -> Option<MatchedReference<'_>> {
     let bytes: &[u8] = sql.as_bytes();
-    let mut index: usize = consume(bytes, skip_whitespace(bytes, keyword_end)?, b'(')?;
-    index = skip_whitespace(bytes, index)?;
+    let mut index: usize = consume(bytes, skip_whitespace(sql, keyword_end), b'(')?;
+    index = skip_whitespace(sql, index);
     let quote: u8 = *bytes
         .get(index)
         .filter(|byte| matches!(byte, b'\'' | b'"'))?;
     let name_start: usize = index + 1;
     let name_end: usize = identifier_end(bytes, name_start)?;
     index = consume(bytes, name_end, quote)?;
-    index = consume(bytes, skip_whitespace(bytes, index)?, b')')?;
+    index = consume(bytes, skip_whitespace(sql, index), b')')?;
     let mut member: Option<&str> = None;
     if matches!(kind, DeclarationReferenceKind::Enum) {
-        index = consume(bytes, skip_whitespace(bytes, index)?, b'.')?;
-        let member_start: usize = skip_whitespace(bytes, index)?;
+        index = consume(bytes, skip_whitespace(sql, index), b'.')?;
+        let member_start: usize = skip_whitespace(sql, index);
         index = identifier_end(bytes, member_start)?;
         member = Some(&sql[member_start..index]);
     }
@@ -70,21 +80,15 @@ fn match_arguments(
     })
 }
 
-/// Python's `\s*` over ASCII; a non-ASCII byte may be Unicode whitespace, so it defers.
-fn skip_whitespace(bytes: &[u8], mut index: usize) -> Option<usize> {
-    while let Some(byte) = bytes.get(index) {
-        if !byte.is_ascii() {
-            return None;
-        }
-        if !matches!(
-            byte,
-            b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c | 0x1c..=0x1f
-        ) {
+/// Python's `\s*`: every character `str.isspace()` accepts.
+fn skip_whitespace(sql: &str, mut index: usize) -> usize {
+    while let Some(character) = sql[index..].chars().next() {
+        if !is_python_space(character) {
             break;
         }
-        index += 1;
+        index += character.len_utf8();
     }
-    Some(index)
+    index
 }
 
 fn consume(bytes: &[u8], index: usize, expected: u8) -> Option<usize> {

crates/sqlbuild-sqltext/src/compiler/_helpers/declaration_references/scanner.rs

@@ -3,21 +3,43 @@
 use crate::compiler::_helpers::declaration_references::reference_syntax::{
     ReferenceSyntax, match_reference,
 };
-use crate::compiler::models::DeclarationReference;
+use crate::compiler::models::{
+    DeclarationReference, DeclarationReferenceScan, DeclarationReferenceStop,
+};
 
 const SPECIAL_BYTES: &[u8] = b"'\"`$@/-";
 
-/// Every `@enum`/`@const` reference in `sql` with code-point offsets, or None when Python decides.
-pub(crate) fn scan_references(sql: &str) -> Option<Vec<DeclarationReference>> {
+/// Where the walk for the next reference start ended.
+enum NextStart {
+    Reference(usize),
+    End,
+    Stop(DeclarationReferenceStop),
+    Deferred,
+}
+
+/// Python's references in `sql` (code points) and its stopping error; None defers to Python.
+pub(crate) fn scan_references(sql: &str) -> Option<DeclarationReferenceScan> {
     let mut references: Vec<DeclarationReference> = Vec::new();
     let mut offsets: CharOffsets = CharOffsets::default();
     let mut cursor: usize = 0;
+    let mut stop: Option<DeclarationReferenceStop> = None;
     while cursor < sql.len() {
-        let Some(start) = next_reference_start(sql, cursor)? else {
-            break;
+        let start: usize = match next_reference_start(sql, cursor) {
+            NextStart::Reference(start) => start,
+            NextStart::End => break,
+            NextStart::Stop(found) => {
+                stop = Some(found);
+                break;
+            }
+            NextStart::Deferred => return None,
         };
-        let ReferenceSyntax::Matched(matched) = match_reference(sql, start) else {
-            return None;
+        let matched = match match_reference(sql, start) {
+            ReferenceSyntax::Matched(matched) => matched,
+            ReferenceSyntax::Malformed(kind) => {
+                stop = Some(DeclarationReferenceStop::Malformed(kind));
+                break;
+            }
+            ReferenceSyntax::NotReference | ReferenceSyntax::Deferred => return None,
         };
         let start_char: usize = offsets.advance(sql, start);
         let end_char: usize = offsets.advance(sql, matched.end);
@@ -30,14 +52,14 @@ pub(crate) fn scan_references(sql: &str) -> Option<Vec<DeclarationReference>> {
         });
         cursor = matched.end;
     }
-    Some(references)
+    Some(DeclarationReferenceScan { references, stop })
 }
 
-/// The next reference start at or after `start`; the outer None means Python decides.
-fn next_reference_start(sql: &str, start: usize) -> Option<Option<usize>> {
+/// The next reference start at or after `start`, as Python's walk finds it.
+fn next_reference_start(sql: &str, start: usize) -> NextStart {
     let rest: &str = &sql[start..];
     if !rest.contains("@enum") && !rest.contains("@const") {
-        return Some(None);
+        return NextStart::End;
     }
     let bytes: &[u8] = sql.as_bytes();
     let mut index: usize = start;
@@ -46,27 +68,35 @@ fn next_reference_start(sql: &str, start: usize) -> Option<Option<usize>> {
         .position(|byte| SPECIAL_BYTES.contains(byte))
     {
         index += offset;
-        index = match bytes[index] {
-            b'\'' | b'"' => quoted_end(bytes, index, true)?,
-            b'`' => quoted_end(bytes, index, false)?,
-            b'$' => dollar_end(bytes, index)?.unwrap_or(index + 1),
-            b'-' if bytes.get(index + 1) == Some(&b'-') => bytes[index..]
-                .iter()
-                .position(|byte| *byte == b'\n')
-                .map_or(bytes.len(), |newline| index + newline + 1),
-            b'/' if bytes.get(index + 1) == Some(&b'*') => {
-                index + 2 + find(&bytes[index + 2..], b"*/")? + 2
-            }
+        let next: Option<usize> = match bytes[index] {
+            b'\'' | b'"' => quoted_end(bytes, index, true),
+            b'`' => quoted_end(bytes, index, false),
+            b'$' => dollar_end(bytes, index).map(|end| end.unwrap_or(index + 1)),
+            b'-' if bytes.get(index + 1) == Some(&b'-') => Some(
+                bytes[index..]
+                    .iter()
+                    .position(|byte| *byte == b'\n')
+                    .map_or(bytes.len(), |newline| index + newline + 1),
+            ),
+            b'/' if bytes.get(index + 1) == Some(&b'*') => match find(&bytes[index + 2..], b"*/") {
+                Some(close) => Some(index + 2 + close + 2),
+                None => return NextStart::Stop(DeclarationReferenceStop::UnclosedBlockComment),
+            },
             b'@' => match match_reference(sql, index) {
-                ReferenceSyntax::NotReference => index + 1,
-                ReferenceSyntax::Matched(_) | ReferenceSyntax::Deferred => {
-                    return Some(Some(index));
+                ReferenceSyntax::NotReference => Some(index + 1),
+                ReferenceSyntax::Matched(_) | ReferenceSyntax::Malformed(_) => {
+                    return NextStart::Reference(index);
                 }
+                ReferenceSyntax::Deferred => return NextStart::Deferred,
             },
-            _ => index + 1,
+            _ => Some(index + 1),
         };
+        match next {
+            Some(next) => index = next,
+            None => return NextStart::Stop(DeclarationReferenceStop::UnclosedQuote),
+        }
     }
-    Some(None)
+    NextStart::End
 }
 
 /// Python's `quoted_text_end_impl` for `'`, `"` (doubled quotes escape) and backticks.

crates/sqlbuild-sqltext/src/compiler/main/declaration_references.rs

@@ -3,10 +3,10 @@
 use rayon::prelude::{IntoParallelRefIterator, ParallelIterator};
 
 use crate::compiler::_helpers::declaration_references::scanner::scan_references;
-use crate::compiler::models::DeclarationReference;
+use crate::compiler::models::DeclarationReferenceScan;
 
-/// Each SQL string's declaration references in order, or None where Python must expand it.
+/// Each SQL string's references and stopping error, or None where Python must expand it.
 #[must_use]
-pub fn scan_declaration_references(sqls: &[String]) -> Vec<Option<Vec<DeclarationReference>>> {
+pub fn scan_declaration_references(sqls: &[String]) -> Vec<Option<DeclarationReferenceScan>> {
     sqls.par_iter().map(|sql| scan_references(sql)).collect()
 }

crates/sqlbuild-sqltext/src/compiler/models.rs

@@ -42,6 +42,24 @@ pub struct DeclarationReference {
     pub end: usize,
 }
 
+/// The error at which Python's `@enum`/`@const` expansion of a string stops.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum DeclarationReferenceStop {
+    /// Quoted or dollar-quoted text before a later reference is never closed.
+    UnclosedQuote,
+    /// A block comment before a later reference is never closed.
+    UnclosedBlockComment,
+    /// `@enum` or `@const` starts a reference its full pattern does not match.
+    Malformed(DeclarationReferenceKind),
+}
+
+/// The references Python resolves in a string, in order, and the error that ends its walk.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct DeclarationReferenceScan {
+    pub references: Vec<DeclarationReference>,
+    pub stop: Option<DeclarationReferenceStop>,
+}
+
 /// A header whose values nest deeper than the parser allows, located in its file.
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct NestingFailure {

crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs

@@ -7,7 +7,9 @@ use crate::compiler::_helpers::sql_interpolation::substitution::{
 use crate::compiler::_helpers::sql_references::extraction::extract;
 use crate::compiler::main::declaration_references::scan_declaration_references;
 use crate::compiler::main::model_header_single_parsing::parse_one;
-use crate::compiler::models::{AuthoredValue, DeclarationReference};
+use crate::compiler::models::{
+    AuthoredValue, DeclarationReferenceKind, DeclarationReferenceScan, DeclarationReferenceStop,
+};
 use crate::compiler::tests::test_types::HeaderNestingTestCase;
 use std::thread;
 
@@ -251,32 +253,52 @@ pub(crate) fn batch_sizes_bound_workers_by_contract() -> bool {
     true
 }
 
-/// Scan one SQL string and spell its references as `kind:name[.member]@start..end, ...`.
+/// Spell a scan as `kind:name[.member]@start..end, ...` plus ` | stop:<error>` when it stops.
 pub(crate) fn scanned_references(sql: &str) -> Option<String> {
-    let references: Vec<DeclarationReference> = scan_declaration_references(&[sql.to_owned()])
+    let scan: DeclarationReferenceScan = scan_declaration_references(&[sql.to_owned()])
         .pop()
         .flatten()?;
-    Some(
-        references
+    let references: String = scan
+        .references
+        .iter()
+        .map(|reference| {
+            let member: String = reference
+                .member
+                .as_ref()
+                .map_or_else(String::new, |member| format!(".{member}"));
+            format!(
+                "{}:{}{member}@{}..{}",
+                reference.kind.keyword(),
+                reference.name,
+                reference.start,
+                reference.end
+            )
+        })
+        .collect::<Vec<String>>()
+        .join(", ");
+    let stop: String = scan.stop.map_or_else(String::new, |stop| {
+        STOP_NAMES
             .iter()
-            .map(|reference| {
-                let member: String = reference
-                    .member
-                    .as_ref()
-                    .map_or_else(String::new, |member| format!(".{member}"));
-                format!(
-                    "{}:{}{member}@{}..{}",
-                    reference.kind.keyword(),
-                    reference.name,
-                    reference.start,
-                    reference.end
-                )
-            })
-            .collect::<Vec<String>>()
-            .join(", "),
-    )
+            .find(|(candidate, _)| *candidate == stop)
+            .map(|(_, name)| format!(" | stop:{name}"))
+            .expect("every stop has a name")
+    });
+    Some(format!("{references}{stop}"))
 }
 
+const STOP_NAMES: [(DeclarationReferenceStop, &str); 4] = [
+    (DeclarationReferenceStop::UnclosedQuote, "quote"),
+    (DeclarationReferenceStop::UnclosedBlockComment, "comment"),
+    (
+        DeclarationReferenceStop::Malformed(DeclarationReferenceKind::Enum),
+        "invalid enum",
+    ),
+    (
+        DeclarationReferenceStop::Malformed(DeclarationReferenceKind::Constant),
+        "invalid const",
+    ),
+];
+
 /// Rust's default spawned-thread stack: bounded nesting must parse on any ordinary thread.
 const ORDINARY_THREAD_STACK_BYTES: usize = 2 * 1024 * 1024;