perf(compile): faster discovery, scope visibility and warm writes

crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs

@@ -376,6 +376,19 @@ fn parse_model_headers(py: Python<'_>, headers: Vec<String>) -> PyResult<Vec<Par
         .collect()
 }
 
+#[pyfunction]
+fn match_model_headers(
+    py: Python<'_>,
+    contents: Vec<String>,
+) -> PyResult<Vec<Option<(usize, usize, usize)>>> {
+    py.compiler_detach(|| {
+        Ok(crate::compiler::main::model_header_matching::match_batch(
+            &contents,
+        ))
+    })
+    .map_err(value_error)
+}
+
 #[pyfunction]
 fn tokenize_model_header(header: &str) -> PyResult<Vec<(u8, String, usize)>> {
     compiler_guard(|| {
@@ -492,6 +505,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
         module
     )?)?;
     module.add_function(wrap_pyfunction!(extract_sql_tests_json, module)?)?;
+    module.add_function(wrap_pyfunction!(match_model_headers, module)?)?;
     module.add_function(wrap_pyfunction!(parse_model_headers, module)?)?;
     module.add_function(wrap_pyfunction!(tokenize_model_header, module)?)?;
     module.add_function(wrap_pyfunction!(substitute_static_project_vars, module)?)?;

crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/matching.rs

@@ -0,0 +1,145 @@
+//! Exact native equivalent of the possessive Python `MODEL(...)` header regular expression.
+
+/// Code-point offsets of one matched header: header start, header end, and SQL start.
+pub(crate) type HeaderMatchOffsets = (usize, usize, usize);
+
+const MODEL_KEYWORD: &[u8] = b"MODEL";
+
+pub(crate) fn match_one(text: &str) -> Option<HeaderMatchOffsets> {
+    let bytes = text.as_bytes();
+    let mut index = skip_whitespace(text, 0);
+    if !bytes[index..].starts_with(MODEL_KEYWORD) {
+        return None;
+    }
+    index = skip_whitespace(text, index + MODEL_KEYWORD.len());
+    if bytes.get(index) != Some(&b'(') {
+        return None;
+    }
+    let header_start = index + 1;
+    let header_end = scan_header_body(text, header_start);
+    if bytes.get(header_end) != Some(&b')') {
+        return None;
+    }
+    index = skip_whitespace(text, header_end + 1);
+    if bytes.get(index) != Some(&b';') {
+        return None;
+    }
+    let sql_start = skip_whitespace(text, index + 1);
+    Some(code_point_offsets(
+        text,
+        header_start,
+        header_end,
+        sql_start,
+    ))
+}
+
+/// Consume the possessive header body and return the byte offset where it stops.
+fn scan_header_body(text: &str, start: usize) -> usize {
+    let bytes = text.as_bytes();
+    let mut index = start;
+    while let Some(&byte) = bytes.get(index) {
+        index = match byte {
+            b'"' | b'\'' => quoted_string_end(text, index, byte)
+                .unwrap_or_else(|| unclosed_quote_end(text, index)),
+            b')' => {
+                let after = skip_whitespace(text, index + 1);
+                if bytes.get(after) == Some(&b';') {
+                    return index;
+                }
+                index + 1
+            }
+            b'\\' => match escape_end(text, index) {
+                Some(end) => end,
+                None => return index,
+            },
+            _ => index + 1,
+        };
+    }
+    index
+}
+
+/// `"(?:[^"\\]|\\.)*+"`: the closing quote's end, or `None` when the string never closes.
+fn quoted_string_end(text: &str, open: usize, quote: u8) -> Option<usize> {
+    let bytes = text.as_bytes();
+    let mut index = open + 1;
+    while let Some(&byte) = bytes.get(index) {
+        if byte == quote {
+            return Some(index + 1);
+        }
+        if byte == b'\\' {
+            index = escape_end(text, index)?;
+        } else {
+            index += 1;
+        }
+    }
+    None
+}
+
+/// `["'](?:[^\\)]|\\.)*+`: an unclosed quote runs to the next `)` or unescapable backslash.
+fn unclosed_quote_end(text: &str, open: usize) -> usize {
+    let bytes = text.as_bytes();
+    let mut index = open + 1;
+    while let Some(&byte) = bytes.get(index) {
+        match byte {
+            b')' => return index,
+            b'\\' => match escape_end(text, index) {
+                Some(end) => index = end,
+                None => return index,
+            },
+            _ => index += 1,
+        }
+    }
+    index
+}
+
+/// `\\.` with DOTALL: a backslash plus one following code point, if any.
+fn escape_end(text: &str, backslash: usize) -> Option<usize> {
+    text[backslash + 1..]
+        .chars()
+        .next()
+        .map(|escaped| backslash + 1 + escaped.len_utf8())
+}
+
+fn skip_whitespace(text: &str, start: usize) -> usize {
+    let mut index = start;
+    for character in text[start..].chars() {
+        if !is_python_whitespace(character) {
+            break;
+        }
+        index += character.len_utf8();
+    }
+    index
+}
+
+/// Python's `str.isspace()`, which `\s` uses for `str` patterns.
+fn is_python_whitespace(character: char) -> bool {
+    matches!(
+        character,
+        '\t'..='\r'
+            | '\u{1c}'..='\u{20}'
+            | '\u{85}'
+            | '\u{a0}'
+            | '\u{1680}'
+            | '\u{2000}'..='\u{200a}'
+            | '\u{2028}'
+            | '\u{2029}'
+            | '\u{202f}'
+            | '\u{205f}'
+            | '\u{3000}'
+    )
+}
+
+fn code_point_offsets(
+    text: &str,
+    header_start: usize,
+    header_end: usize,
+    sql_start: usize,
+) -> HeaderMatchOffsets {
+    if text.is_ascii() {
+        return (header_start, header_end, sql_start);
+    }
+    let header_start_chars = text[..header_start].chars().count();
+    let header_end_chars = header_start_chars + text[header_start..header_end].chars().count();
+    let sql_start_chars = header_end_chars + text[header_end..sql_start].chars().count();
+    (header_start_chars, header_end_chars, sql_start_chars)
+}

crates/sqlbuild-rules-native/src/compiler/_helpers/model_headers/mod.rs

@@ -1 +1,2 @@
+pub(crate) mod matching;
 pub(crate) mod tokenization;

crates/sqlbuild-rules-native/src/compiler/main/mod.rs

@@ -1,3 +1,4 @@
+pub(crate) mod model_header_matching;
 pub(crate) mod model_header_parsing;
 pub(crate) mod model_header_tokenizing;
 pub(crate) mod sql_interpolation;

crates/sqlbuild-rules-native/src/compiler/main/model_header_matching.rs

@@ -0,0 +1,8 @@
+use crate::compiler::_helpers::model_headers::matching::{self, HeaderMatchOffsets};
+
+pub(crate) fn match_batch(contents: &[String]) -> Vec<Option<HeaderMatchOffsets>> {
+    contents
+        .iter()
+        .map(|text| matching::match_one(text))
+        .collect()
+}

crates/sqlbuild-rules-native/src/compiler/tests.rs

@@ -1,5 +1,7 @@
 #[path = "tests/helpers.rs"]
 mod helpers;
+#[path = "tests/test_model_header_matching.rs"]
+mod model_header_matching;
 #[path = "tests/test_model_header_tokenization.rs"]
 mod model_header_tokenization;
 #[path = "tests/test_sql_interpolation.rs"]

crates/sqlbuild-rules-native/src/compiler/tests/test_model_header_matching.rs

@@ -0,0 +1,72 @@
+use crate::compiler::main::model_header_matching::match_batch;
+use crate::compiler::tests::test_types::ModelHeaderMatchTestCase;
+
+#[test]
+fn given_model_files_when_matching_headers_then_offsets_follow_the_python_pattern() {
+    let test_cases = [
+        ModelHeaderMatchTestCase {
+            description: "simple header",
+            contents: "MODEL (name orders);\nSELECT 1",
+            expected_offsets: Some((7, 18, 21)),
+        },
+        ModelHeaderMatchTestCase {
+            description: "leading whitespace and spaced terminator",
+            contents: " \n MODEL\t( a ) \n ; \n SELECT 1",
+            expected_offsets: Some((10, 13, 21)),
+        },
+        ModelHeaderMatchTestCase {
+            description: "closing parenthesis not followed by a semicolon stays in the header",
+            contents: "MODEL (columns (id INT));SELECT 1",
+            expected_offsets: Some((7, 23, 25)),
+        },
+        ModelHeaderMatchTestCase {
+            description: "quoted closing parenthesis and semicolon stay in the header",
+            contents: "MODEL (description 'a);b');SELECT 1",
+            expected_offsets: Some((7, 25, 27)),
+        },
+        ModelHeaderMatchTestCase {
+            description: "unclosed quote runs to the next parenthesis",
+            contents: "MODEL (description 'open);SELECT 1",
+            expected_offsets: Some((7, 24, 26)),
+        },
+        ModelHeaderMatchTestCase {
+            description: "escaped quote inside a string",
+            contents: "MODEL (description \"a\\\"b\");SELECT 1",
+            expected_offsets: Some((7, 25, 27)),
+        },
+        ModelHeaderMatchTestCase {
+            description: "non ascii offsets are code points",
+            contents: "MODEL (description 'caf\u{e9}')\u{a0};\u{3000}SELECT 1",
+            expected_offsets: Some((7, 25, 29)),
+        },
+        ModelHeaderMatchTestCase {
+            description: "missing terminator",
+            contents: "MODEL (name orders)\nSELECT 1",
+            expected_offsets: None,
+        },
+        ModelHeaderMatchTestCase {
+            description: "trailing lone backslash",
+            contents: "MODEL (name orders\\",
+            expected_offsets: None,
+        },
+        ModelHeaderMatchTestCase {
+            description: "lowercase keyword",
+            contents: "model (name orders);",
+            expected_offsets: None,
+        },
+        ModelHeaderMatchTestCase {
+            description: "empty file",
+            contents: "",
+            expected_offsets: None,
+        },
+    ];
+
+    for test_case in test_cases {
+        let actual = match_batch(&[test_case.contents.to_string()])[0];
+        assert_eq!(
+            actual, test_case.expected_offsets,
+            "{}",
+            test_case.description
+        );
+    }
+}

crates/sqlbuild-rules-native/src/compiler/tests/test_types.rs

@@ -86,3 +86,9 @@ pub(crate) struct RelationMarkerOracleTestCase {
     pub(crate) dialect: &'static str,
     pub(crate) expected_mismatches: &'static [&'static str],
 }
+
+pub(crate) struct ModelHeaderMatchTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) contents: &'static str,
+    pub(crate) expected_offsets: Option<(usize, usize, usize)>,
+}

src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md

@@ -50,7 +50,10 @@ line_width = 100
 `MODEL`, `SCENARIO`, `FUNCTION` and `HOOK` headers must carry a non-empty `description`;
 `sqb format --check` fails with `description-present` when one is missing, so pre-commit catches it
 before compile does (see [Descriptions](../concepts/models/configuration.md#descriptions)). A model
-described by `[defaults]`, `[path_defaults]` or its `model_schema` passes, exactly as it compiles.
+described by its `model_schema`, or by the one `[path_defaults]` entry that applies to it, passes,
+exactly as it compiles. `[defaults]` does not accept `description` (`D001`), and only the nearest
+matching path default applies (see
+[Which path default applies](../concepts/project-configuration.md#which-path-default-applies)).
 Format does not render templates, so a templated description that renders empty passes format and
 is reported by `sqb compile`.
 Plain `sqb format` still formats the file and exits non-zero, like any other remaining fault.

src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md

@@ -386,14 +386,52 @@ replay_on_change = "full"
 ```
 
 Path matching uses the path below `models/`. A model at `models/staging/stg_orders.sql` matches the
-`staging` path default.
+`staging` path default. A key matches its folder and every folder below it. In a key, `*` matches one
+folder name and `**` matches any number of folders, so `"**"` matches every model.
+
+### Which path default applies
+
+Each model uses at most one path default: the nearest matching key. Path defaults are not merged, so
+a model gets nothing from a broader key when a more specific key also matches it.
+
+1. A key without wildcards beats every wildcard key. Among those, the deepest key wins, so
+   `staging/orders` beats `staging`.
+2. When only wildcard keys match, the most specific wins: more literal folder names first, then more
+   `*` segments, then fewer `**` segments. Two equally specific matches fail compilation.
+
+A broad `"**"` default therefore applies only to models that no other key matches. With the
+configuration below, `models/orders.sql` gets the `"**"` description, but
+`models/staging/stg_customers.sql` uses only the `staging` entry. It has no description and fails
+with `P010`:
+
+```toml
+[path_defaults."**"]
+description = "Order analytics model."
+
+[path_defaults.staging]
+materialized = "view"
+```
+
+Descriptions belong to individual models, so put each one in the `MODEL()` header, or in the
+`description` of the model's `model_schema`. `[defaults]` rejects `description` with `D001`. If a
+folder of models really shares one description, every key those models can select must set it:
+
+```toml
+[path_defaults."**"]
+description = "Order analytics model."
+
+[path_defaults.staging]
+materialized = "view"
+description = "Staging model."
+```
 
 ### Config layering order
 
 Configuration is layered in this order, with later layers overriding earlier ones:
 
 1. **Project defaults** (`defaults`)
-2. **Path defaults** (`path_defaults`) - if the model's path matches
+2. **Path defaults** (`path_defaults`) - the one nearest matching entry, if any (see
+   [Which path default applies](#which-path-default-applies))
 3. **MODEL() header** - the model's own config
 
 Most keys are overridden by the more specific layer, but three merge instead:

src/sqlbuild/_native.pyi

@@ -31,6 +31,7 @@ def load_config_json(project_dir: str) -> str: ...
 def selected_codes_json(request_json: str) -> str: ...
 def render_owned_skill(content: str, input_fingerprint: str) -> str: ...
 def skill_freshness(content: str | None, input_fingerprint: str) -> str: ...
+def match_model_headers(contents: list[str]) -> list[tuple[int, int, int] | None]: ...
 def parse_model_headers(
     headers: list[str],
 ) -> list[tuple[dict[str, object] | None, list[tuple[str, int, int]] | None, str | None]]: ...