feat(native): attach SQL tests, scenarios and functions natively behind the preview engine

crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs

@@ -58,7 +58,7 @@ struct TestRequest {
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Serialize)]
-struct Cte(String, String);
+pub(crate) struct Cte(pub(crate) String, pub(crate) String);
 
 #[derive(Debug, PartialEq, Eq, Serialize)]
 #[serde(tag = "kind", rename_all = "snake_case")]
@@ -105,6 +105,12 @@ pub(crate) fn extract_batch_json(request_json: &str) -> Result<String, String> {
 }
 
 fn extract_ctes(sql: &str, file: &str) -> Result<Vec<Cte>, String> {
+    extract_ctes_with_quoting(sql, file).map(|(ctes, _)| ctes)
+}
+
+/// The top-level CTEs and whether any CTE name was a quoted identifier.
+pub(crate) fn extract_ctes_with_quoting(sql: &str, file: &str) -> Result<(Vec<Cte>, bool), String> {
+    let mut quoted = false;
     let mut index = skip_ignorable(sql, 0)?;
     index = consume_keyword(sql, index, "WITH").ok_or_else(|| {
         format!("SQL test '{file}' must declare mock CTEs and one __expected__<model> CTE in a top-level WITH clause")
@@ -116,6 +122,7 @@ fn extract_ctes(sql: &str, file: &str) -> Result<Vec<Cte>, String> {
     let mut ctes: Vec<Cte> = Vec::new();
     let mut seen: HashSet<String> = HashSet::new();
     loop {
+        quoted |= matches!(byte_at(sql, index), Some(b'"' | b'`'));
         let (name, end) = read_identifier(sql, index)
             .ok_or_else(|| format!("SQL test '{file}' expected a CTE name"))?;
         if !seen.insert(name.clone()) {
@@ -145,7 +152,7 @@ fn extract_ctes(sql: &str, file: &str) -> Result<Vec<Cte>, String> {
             "SQL test '{file}' must end after its CTEs; only an optional ceremonial top-level `SELECT 1` may follow them"
         ));
     }
-    Ok(ctes)
+    Ok((ctes, quoted))
 }
 
 fn classify(ctes: Vec<Cte>, file: &str, mode: &str) -> Result<Classified, String> {
@@ -568,7 +575,7 @@ fn implicit_projection_alias(expression: &str) -> Option<String> {
     (end == expression.len()).then_some(alias)
 }
 
-fn validate_independence(ctes: &[Cte], file: &str) -> Result<(), String> {
+pub(crate) fn validate_independence(ctes: &[Cte], file: &str) -> Result<(), String> {
     let names: HashMap<String, String> = ctes
         .iter()
         .map(|cte| (cte.0.to_lowercase(), cte.0.clone()))

crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs

@@ -9,3 +9,4 @@ pub mod markers;
 pub mod planning;
 pub mod relation_markers;
 pub mod rendering;
+pub mod scenario_extraction;

crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs

@@ -0,0 +1,101 @@
+//! Python's `extract_sql_scenario_ctes` for scenarios its top-level scanner reads exactly.
+
+use serde::Serialize;
+
+use crate::compiler::_helpers::sql_tests::extraction::{
+    Cte, extract_ctes_with_quoting, validate_independence,
+};
+
+/// Characters whose Python case mapping reaches ASCII, which Python's keyword match would accept.
+const CASE_MAPPED_TO_ASCII: [char; 20] = [
+    '\u{df}', '\u{130}', '\u{131}', '\u{149}', '\u{17f}', '\u{1f0}', '\u{1e96}', '\u{1e97}',
+    '\u{1e98}', '\u{1e99}', '\u{1e9a}', '\u{1e9e}', '\u{212a}', '\u{fb00}', '\u{fb01}', '\u{fb02}',
+    '\u{fb03}', '\u{fb04}', '\u{fb05}', '\u{fb06}',
+];
+
+#[derive(Debug, Default, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct ClassifiedScenario {
+    authored: Vec<Cte>,
+    expected: Vec<Cte>,
+    assertions: Vec<Cte>,
+    source_fixtures: Vec<String>,
+    ref_fixtures: Vec<String>,
+    seed_fixtures: Vec<String>,
+    dbt_ref_fixtures: Vec<String>,
+    expected_models: Vec<String>,
+    assertion_names: Vec<String>,
+}
+
+pub(crate) fn extract_scenario_json(sql: &str, file: &str) -> Result<Option<String>, String> {
+    if sql.chars().any(|character| {
+        matches!(character, '\u{1c}'..='\u{1f}') || CASE_MAPPED_TO_ASCII.contains(&character)
+    }) {
+        return Ok(None);
+    }
+    let Ok((ctes, quoted)) = extract_ctes_with_quoting(sql, file) else {
+        return Ok(None);
+    };
+    if quoted || !ctes.iter().all(|cte| is_plain_identifier(&cte.0)) {
+        return Ok(None);
+    }
+    if validate_independence(&ctes, file).is_err() {
+        return Ok(None);
+    }
+    let Some(scenario) = classify(ctes) else {
+        return Ok(None);
+    };
+    serde_json::to_string(&scenario)
+        .map(Some)
+        .map_err(|error| error.to_string())
+}
+
+/// Python's `_read_identifier` reads only `[A-Za-z_][A-Za-z0-9_]*` the same way.
+fn is_plain_identifier(name: &str) -> bool {
+    let mut bytes = name.bytes();
+    bytes
+        .next()
+        .is_some_and(|first| first.is_ascii_alphabetic() || first == b'_')
+        && bytes.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_')
+}
+
+fn classify(ctes: Vec<Cte>) -> Option<ClassifiedScenario> {
+    let mut scenario: ClassifiedScenario = ClassifiedScenario::default();
+    for cte in ctes {
+        let name: String = cte.0.clone();
+        if let Some(value) = name.strip_prefix("__source__") {
+            scenario.source_fixtures.push(required(value)?);
+            scenario.authored.push(cte);
+        } else if let Some(value) = name.strip_prefix("__ref__") {
+            scenario.ref_fixtures.push(required(value)?);
+            scenario.authored.push(cte);
+        } else if let Some(value) = name.strip_prefix("__seed__") {
+            scenario.seed_fixtures.push(required(value)?);
+            scenario.authored.push(cte);
+        } else if let Some(value) = name.strip_prefix("__dbt_ref__") {
+            scenario.dbt_ref_fixtures.push(required(value)?);
+            scenario.authored.push(cte);
+        } else if let Some(value) = name.strip_prefix("__expected__") {
+            scenario.expected_models.push(required(value)?);
+            scenario.expected.push(cte);
+        } else if let Some(value) = name.strip_prefix("__assert__") {
+            scenario.assertion_names.push(required(value)?);
+            scenario.assertions.push(cte);
+        } else if name.starts_with("__macro__") {
+            return None;
+        } else {
+            scenario.authored.push(cte);
+        }
+    }
+    let has_fixture: bool = !(scenario.source_fixtures.is_empty()
+        && scenario.ref_fixtures.is_empty()
+        && scenario.seed_fixtures.is_empty()
+        && scenario.dbt_ref_fixtures.is_empty());
+    let has_check: bool =
+        !(scenario.expected_models.is_empty() && scenario.assertion_names.is_empty());
+    (has_fixture && has_check).then_some(scenario)
+}
+
+fn required(value: &str) -> Option<String> {
+    (!value.is_empty()).then(|| value.to_owned())
+}

crates/sqlbuild-analysis/src/compiler/main/mod.rs

@@ -1,3 +1,4 @@
+pub mod sql_scenario_extraction;
 pub mod sql_test_chain_resolution;
 pub mod sql_test_difference_sampling;
 pub mod sql_test_extraction;

crates/sqlbuild-analysis/src/compiler/main/sql_scenario_extraction.rs

@@ -0,0 +1,6 @@
+use crate::compiler::_helpers;
+
+/// Python's extracted scenario CTEs as JSON, or `None` where Python must extract the scenario.
+pub fn extract_scenario_json(sql: &str, file_label: &str) -> Result<Option<String>, String> {
+    _helpers::sql_tests::scenario_extraction::extract_scenario_json(sql, file_label)
+}

crates/sqlbuild-analysis/src/compiler/tests.rs

@@ -1,5 +1,7 @@
 #[path = "tests/helpers.rs"]
 mod helpers;
+#[path = "tests/test_sql_scenario_extraction.rs"]
+mod sql_scenario_extraction;
 #[path = "tests/test_sql_test_cte_rename.rs"]
 mod sql_test_cte_rename;
 #[path = "tests/test_sql_test_cte_slices.rs"]

crates/sqlbuild-analysis/src/compiler/tests/test_sql_scenario_extraction.rs

@@ -0,0 +1,46 @@
+use crate::compiler::main::sql_scenario_extraction::extract_scenario_json;
+use crate::compiler::tests::test_types::ScenarioExtractionTestCase;
+
+#[test]
+fn given_scenarios_when_extracting_natively_then_python_payloads_or_deferrals_are_returned() {
+    let test_cases = [
+        ScenarioExtractionTestCase {
+            description: "fixtures, helpers, expected results and assertions are classified",
+            sql: "WITH helper AS (SELECT 1 AS id), __source__raw_orders AS (SELECT id FROM helper), \
+                  __expected__orders AS (SELECT 1 AS id), __assert__positive AS (SELECT 1) SELECT 1",
+            expected_json: Some(
+                r#"{"authored":[["helper","SELECT 1 AS id"],["__source__raw_orders","SELECT id FROM helper"]],"expected":[["__expected__orders","SELECT 1 AS id"]],"assertions":[["__assert__positive","SELECT 1"]],"sourceFixtures":["raw_orders"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":["orders"],"assertionNames":["positive"]}"#,
+            ),
+        },
+        ScenarioExtractionTestCase {
+            description: "a macro mock defers so Python raises",
+            sql: "WITH __source__raw AS (SELECT 1), __macro__tidy AS (SELECT 'x'), \
+                  __expected__orders AS (SELECT 1 AS id) SELECT 1",
+            expected_json: None,
+        },
+        ScenarioExtractionTestCase {
+            description: "a quoted CTE name defers to Python's scanner fallback",
+            sql: "WITH \"__source__raw\" AS (SELECT 1), __expected__orders AS (SELECT 1 AS id)",
+            expected_json: None,
+        },
+        ScenarioExtractionTestCase {
+            description: "a keyword Python would match by case mapping defers",
+            sql: "w\u{131}th __source__raw AS (SELECT 1), __expected__orders AS (SELECT 1 AS id)",
+            expected_json: None,
+        },
+        ScenarioExtractionTestCase {
+            description: "a scenario without fixtures defers so Python raises",
+            sql: "WITH __expected__orders AS (SELECT 1 AS id) SELECT 1",
+            expected_json: None,
+        },
+    ];
+
+    for test_case in test_cases {
+        assert_eq!(
+            extract_scenario_json(test_case.sql, "tests/scenarios/orders.sql"),
+            Ok(test_case.expected_json.map(str::to_owned)),
+            "{}",
+            test_case.description
+        );
+    }
+}

crates/sqlbuild-analysis/src/compiler/tests/test_types.rs

@@ -131,3 +131,9 @@ pub(crate) struct HelperReferenceTestCase {
     pub(crate) expected_fragments: &'static [&'static str],
     pub(crate) expected_absent_fragments: &'static [&'static str],
 }
+
+pub(crate) struct ScenarioExtractionTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) sql: &'static str,
+    pub(crate) expected_json: Option<&'static str>,
+}

crates/sqlbuild-attachments/Cargo.toml

@@ -12,3 +12,4 @@ workspace = true
 
 [dependencies]
 sqlbuild-sqltext.workspace = true
+sqlbuild-core.workspace = true

crates/sqlbuild-attachments/src/ceremonial_select/_helpers/boundaries.rs

@@ -0,0 +1,35 @@
+//! Python's code boundary pattern and `WITH` keyword check.
+
+/// Python's boundary pattern: parentheses, `/*`, quotes, `$` and the line comment prefixes.
+pub(crate) fn next_boundary(bytes: &[u8], start: usize, prefixes: &[String]) -> Option<usize> {
+    (start..bytes.len()).find(|index| is_boundary(&bytes[*index..], prefixes))
+}
+
+fn is_boundary(rest: &[u8], prefixes: &[String]) -> bool {
+    if matches!(rest[0], b'(' | b')' | b'\'' | b'"' | b'`' | b'$') || rest.starts_with(b"/*") {
+        return true;
+    }
+    for prefix in prefixes {
+        if rest.starts_with(prefix.as_bytes()) {
+            return true;
+        }
+    }
+    false
+}
+
+/// Whether `WITH` (any case) starts at `first` as a whole word; None for non-ASCII text.
+pub(crate) fn starts_with_keyword(sql: &str, first: usize) -> Option<bool> {
+    let mut characters = sql[first..].chars();
+    let keyword: String = characters.by_ref().take(4).collect();
+    if !keyword.is_ascii() {
+        return None;
+    }
+    if !keyword.eq_ignore_ascii_case("WITH") {
+        return Some(false);
+    }
+    match characters.next() {
+        None => Some(true),
+        Some(next) if !next.is_ascii() => None,
+        Some(next) => Some(!(next.is_ascii_alphanumeric() || next == '_')),
+    }
+}

crates/sqlbuild-attachments/src/ceremonial_select/_helpers/mod.rs

@@ -0,0 +1,2 @@
+pub(crate) mod boundaries;
+pub(crate) mod segments;

crates/sqlbuild-attachments/src/ceremonial_select/_helpers/segments.rs

@@ -0,0 +1,51 @@
+//! Python's code segment bounds and statement terminator characters.
+
+/// Python's `" \t\r\n\f\v;"`.
+pub(crate) fn is_terminator(byte: u8) -> bool {
+    matches!(byte, b' ' | b'\t' | b'\r' | b'\n' | 0x0c | 0x0b | b';')
+}
+
+/// `sql` without trailing terminator characters.
+pub(crate) fn trimmed_end(sql: &[u8], start: usize, end: usize) -> usize {
+    let mut index: usize = end;
+    while index > start && is_terminator(sql[index - 1]) {
+        index -= 1;
+    }
+    index
+}
+
+/// The first and last code bytes so far, extended by the code segment `start..end`.
+pub(crate) fn segment_bounds(
+    sql: &[u8],
+    start: usize,
+    end: usize,
+    bounds: Option<(usize, usize)>,
+) -> Option<(usize, usize)> {
+    let stripped_end: usize = trimmed_end(sql, start, end);
+    let leading: usize = sql[start..stripped_end]
+        .iter()
+        .position(|byte| !is_terminator(*byte))?;
+    let first: usize = bounds.map_or(start + leading, |(first, _)| first);
+    Some((first, stripped_end - 1))
+}
+
+/// Python's `_may_end_with_comment` over the body without trailing terminators.
+pub(crate) fn may_end_with_comment(tail: &[u8], prefixes: &[String]) -> bool {
+    let line_start: usize = tail
+        .iter()
+        .rposition(|byte| *byte == b'\n')
+        .map_or(0, |index| index + 1);
+    let last_line: &[u8] = &tail[line_start..];
+    if tail.ends_with(b"*/") {
+        return true;
+    }
+    for prefix in prefixes {
+        if last_line
+            .windows(prefix.len())
+            .any(|window| window == prefix.as_bytes())
+        {
+            return true;
+        }
+    }
+    false
+}

crates/sqlbuild-attachments/src/ceremonial_select/main/mod.rs

@@ -0,0 +1 @@
+pub mod omitted_select_offset;