fix(compile): linear missing-description locations and comment-tolerant expected fixtures (#924)

* perf(compile): index YAML entry lines once for missing-description locations

Locating each undescribed seed or source rescanned its whole YAML file with
a per-name regex, so error-heavy compiles grew quadratically with entries per
file. Index every entry name once per file and look names up from that index,
preserving the outermost-entry and pattern semantics exactly.

* fix(test): ignore comments around set-operation branches in SQL-test fixtures

A comment before or after UNION ALL in an __expected__ fixture stayed
attached to the neighbouring branch, so the projection scanners read the
comment as an alias and reported a false P001 branch mismatch. Branch and
projection splits now drop surrounding comments, alias detection ignores
trailing comments, the Python fallback splitter follows the adapter's
lexical syntax, and Polyglot's annotated comment wrapper is unwrapped.

Author: kvlonge

crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs

@@ -512,7 +512,7 @@ fn projection_name(expression: &str, file: &str, label: &str) -> Result<String,
     if let Some(position) = find_last_top_level_keyword(expression, "AS")? {
         let alias_start = skip_ignorable(expression, position + 2)?;
         if let Some((alias, end)) = read_identifier(expression, alias_start)
-            && expression[end..].trim().is_empty()
+            && skip_ignorable(expression, end)? == expression.len()
         {
             return Ok(alias);
         }
@@ -880,8 +880,34 @@ fn scan_code<T>(
     Ok(None)
 }
 
-fn non_empty_trimmed(value: &str) -> Option<&str> {
-    Some(value.trim()).filter(|trimmed| !trimmed.is_empty())
+/// Return `value` without surrounding whitespace and comments, or `None` when no code remains.
+fn non_empty_trimmed(value: &str) -> Result<Option<&str>, String> {
+    if !value.contains("--") && !value.contains("/*") {
+        return Ok(Some(value.trim()).filter(|trimmed| !trimmed.is_empty()));
+    }
+    let start = skip_ignorable(value, 0)?;
+    let mut end = start;
+    let mut index = start;
+    while index < value.len() {
+        if let Some(comment) = comment_end(value.as_bytes(), index)
+            .map_err(|error| scan_error_message(error, "SQL test"))?
+        {
+            index = comment;
+            continue;
+        }
+        let next = skip_non_code(value, index)?;
+        if next != index {
+            end = next;
+            index = next;
+            continue;
+        }
+        let width = char_len(value, index);
+        if !value[index..].starts_with(char::is_whitespace) {
+            end = index + width;
+        }
+        index += width;
+    }
+    Ok(Some(&value[start..end]).filter(|trimmed| !trimmed.is_empty()))
 }
 
 /// Split on top-level `UNION`, `INTERSECT` and `EXCEPT`, each with an optional `ALL`/`DISTINCT`.
@@ -895,7 +921,7 @@ pub(crate) fn split_set_operations(sql: &str) -> Result<Vec<&str>, String> {
         let Some(end) = set_operator_end(sql, index)? else {
             return Ok(CodeStep::Advance);
         };
-        values.extend(non_empty_trimmed(&sql[start..index]));
+        values.extend(non_empty_trimmed(&sql[start..index])?);
         let mut resume = skip_ignorable(sql, end)?;
         if let Some(quantifier_end) =
             consume_keyword(sql, resume, "ALL").or_else(|| consume_keyword(sql, resume, "DISTINCT"))
@@ -905,7 +931,7 @@ pub(crate) fn split_set_operations(sql: &str) -> Result<Vec<&str>, String> {
         start = resume;
         Ok(CodeStep::Resume(resume))
     })?;
-    values.extend(non_empty_trimmed(&sql[start..]));
+    values.extend(non_empty_trimmed(&sql[start..])?);
     Ok(values)
 }
 
@@ -927,12 +953,12 @@ pub(crate) fn split_top_level(sql: &str, separator: u8) -> Result<Vec<&str>, Str
     let mut start = 0;
     scan_code::<()>(sql, 0, |index, depth| {
         if depth == 0 && byte_at(sql, index) == Some(separator) {
-            values.extend(non_empty_trimmed(&sql[start..index]));
+            values.extend(non_empty_trimmed(&sql[start..index])?);
             start = index + 1;
         }
         Ok(CodeStep::Advance)
     })?;
-    values.extend(non_empty_trimmed(&sql[start..]));
+    values.extend(non_empty_trimmed(&sql[start..])?);
     Ok(values)
 }
 

crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_extraction.rs

@@ -91,12 +91,26 @@ fn given_expected_cte_sql_when_scanning_top_level_then_set_operations_commas_and
             ]),
             expected_commas: Ok(vec![
                 "SELECT 'x, UNION (' AS a",
-                "/* UNION , FROM */ \"b)\"",
+                "\"b)\"",
                 "`c,d`",
                 "f(1, (SELECT 2 FROM t UNION SELECT 3)) -- , FROM\nFROM t",
             ]),
             expected_from: Ok(Some(107)),
         },
+        TopLevelScanTestCase {
+            description: "comments around branches and projections are not code",
+            sql: "-- lead\nSELECT 1 AS a, /* b */ 2 AS b -- first\n/* x */ UNION ALL -- second\nSELECT '--' AS a, 3 AS b /* end */",
+            expected_unions: Ok(vec![
+                "SELECT 1 AS a, /* b */ 2 AS b",
+                "SELECT '--' AS a, 3 AS b",
+            ]),
+            expected_commas: Ok(vec![
+                "SELECT 1 AS a",
+                "2 AS b -- first\n/* x */ UNION ALL -- second\nSELECT '--' AS a",
+                "3 AS b",
+            ]),
+            expected_from: Ok(None),
+        },
         TopLevelScanTestCase {
             description: "keyword boundaries and empty pieces",
             sql: ",SELECT reunion, union_id,, x FROMAGE UNION",

scripts/cold_compile_performance/_helpers/diagnostic_project.py

@@ -132,3 +132,34 @@ def write_single_model_diagnostic_project(*, project_dir: Path, diagnostics: int
         + ",\n".join(projections)
         + '\nFROM __ref("orders") AS o\n'
     )
+
+
+def write_undescribed_project(*, project_dir: Path, model_count: int, width: int = 6) -> None:
+    """Generate models, seeds, and one large source file whose resources all lack descriptions."""
+    project_dir.mkdir(parents=True, exist_ok=True)
+    for folder in ("models", "sources", "seeds"):
+        (project_dir / folder).mkdir(exist_ok=True)
+    (project_dir / "sqlbuild_project.toml").write_text(
+        'name = "undescribed_orders"\nadapter = "duckdb"\n[rules]\nselect = []\n'
+    )
+    columns: list[str] = [f"amount_{index:03}" for index in range(width)]
+    sources: list[str] = ["sources:"]
+    seeds: list[str] = ["seeds:"]
+    for index in range(model_count):
+        sources.extend(
+            (f"  - name: orders_{index:05}", "    schema: main", f"    table: orders_{index:05}")
+        )
+        sources.append("    columns:")
+        sources.extend(f"      - name: {column}\n        type: INTEGER" for column in columns)
+        (project_dir / "models" / f"order_totals_{index:05}.sql").write_text(
+            "MODEL (materialized view);\n"
+            f'SELECT {", ".join(columns)} FROM __source("orders_{index:05}")\n'
+        )
+        if index % 10 == 0:
+            seeds.append(f"  - name: products_{index:05}\n    columns:")
+            seeds.extend(f"      - name: {column}\n        type: INTEGER" for column in columns)
+            (project_dir / "seeds" / f"products_{index:05}.csv").write_text(
+                ",".join(columns) + "\n" + ",".join("1" for _ in columns) + "\n"
+            )
+    (project_dir / "sources" / "orders.yml").write_text("\n".join(sources) + "\n")
+    (project_dir / "seeds" / "products.yml").write_text("\n".join(seeds) + "\n")

src/sqlbuild/compiler/compile/_helpers/analysis/tests.py

@@ -15,14 +15,19 @@
     split_set_operation_branches,
 )
 from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql
+from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax
 
 _POLYGLOT_VALUES_EXPRESSION_NAME: str = "Values"
 _POLYGLOT_VALUES_SET_ALIAS: str = "_values"
 _SCAN_CONTEXT: str = "SQL test"
 
 
 def extract_expected_branch_column_names_with_sql_analysis(
-    *, sql: str, file_label: str, label: str = "__expected__<model>"
+    *,
+    sql: str,
+    file_label: str,
+    syntax: SqlLexicalSyntax,
+    label: str = "__expected__<model>",
 ) -> tuple[tuple[str, ...], ...] | None:
     """Return expected SELECT branch names using required Polyglot analysis."""
 
@@ -32,17 +37,23 @@ def extract_expected_branch_column_names_with_sql_analysis(
         parsed_expression: Any = polyglot_module.parse_one(sql, dialect="generic")
     except polyglot_module.PolyglotError:
         return _extract_unparsed_branch_names(
-            sql=sql, polyglot_module=polyglot_module, file_label=file_label, label=label
+            sql=sql,
+            polyglot_module=polyglot_module,
+            file_label=file_label,
+            label=label,
+            syntax=syntax,
         )
     return _extract_branch_names(expression=parsed_expression, file_label=file_label, label=label)
 
 
 def _extract_unparsed_branch_names(
-    *, sql: str, polyglot_module: Any, file_label: str, label: str
+    *, sql: str, polyglot_module: Any, file_label: str, label: str, syntax: SqlLexicalSyntax
 ) -> tuple[tuple[str, ...], ...] | None:
     """Parse each top-level set-operation branch separately when the whole query cannot parse."""
 
-    branches: tuple[str, ...] = split_set_operation_branches(sql=sql, context=_SCAN_CONTEXT)
+    branches: tuple[str, ...] = split_set_operation_branches(
+        sql=sql, context=_SCAN_CONTEXT, syntax=syntax
+    )
     if len(branches) <= 1:
         return None
     try:

src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py

@@ -762,12 +762,14 @@ def _extract_expected_branch_column_names(
 ) -> tuple[tuple[str, ...], ...]:
     sql_analysis_column_names: tuple[tuple[str, ...], ...] | None = (
         extract_expected_branch_column_names_with_sql_analysis(
-            sql=sql, file_label=file_label, label=label
+            sql=sql, file_label=file_label, label=label, syntax=syntax
         )
     )
     if sql_analysis_column_names is not None:
         return sql_analysis_column_names
-    branches: tuple[str, ...] = split_set_operation_branches(sql=sql, context=_CONTEXT)
+    branches: tuple[str, ...] = split_set_operation_branches(
+        sql=sql, context=_CONTEXT, syntax=syntax
+    )
     return tuple(
         _extract_expected_select_column_names(
             branch_sql=branch, file_label=file_label, label=label, syntax=syntax
@@ -837,9 +839,13 @@ def _extract_expected_projection_name(
     alias_name: str | None = _extract_as_alias(expression=expression, syntax=syntax)
     if alias_name is not None:
         return alias_name
-    stripped_expression: str = expression.strip()
-    if _is_simple_identifier(stripped_expression):
-        return stripped_expression
+    start: int = _skip_ignorable(sql=expression, start=0, syntax=syntax)
+    if start < len(expression) and is_identifier_start(expression[start]):
+        name: str
+        end: int
+        name, end = _read_identifier(sql=expression, start=start, file_label="projection")
+        if _skip_ignorable(sql=expression, start=end, syntax=syntax) == len(expression):
+            return name
     raise CompileInputError(
         f"SQL test '{file_label}' must alias every non-trivial {label} projection"
     )
@@ -862,17 +868,11 @@ def _extract_as_alias(*, expression: str, syntax: SqlLexicalSyntax) -> str | Non
                 start=alias_index,
                 file_label="projection",
             )
-            if not expression[alias_end:].strip():
+            if _skip_ignorable(sql=expression, start=alias_end, syntax=syntax) == len(expression):
                 last_alias_name = alias_name
     return last_alias_name
 
 
-def _is_simple_identifier(value: str) -> bool:
-    if not value or not is_identifier_start(value[0]):
-        return False
-    return all(is_identifier_character(character) for character in value[1:])
-
-
 def _contains_select_star(*, sql: str, syntax: SqlLexicalSyntax) -> bool:
     index: int
     for index, _depth in iter_code_positions(sql=sql, context=_CONTEXT, syntax=syntax):

src/sqlbuild/compiler/compile/constants.py

@@ -96,7 +96,7 @@
 )
 POLYGLOT_SELECT_EXPRESSION_NAME: str = "Select"
 POLYGLOT_COLUMN_EXPRESSION_NAME: str = "Column"
-POLYGLOT_WRAPPER_EXPRESSION_NAMES: frozenset[str] = frozenset({"Subquery", "Paren"})
+POLYGLOT_WRAPPER_EXPRESSION_NAMES: frozenset[str] = frozenset({"Annotated", "Subquery", "Paren"})
 
 TABLE_FUNCTION_RETURN_KEYS: frozenset[str] = frozenset({"table"})
 

src/sqlbuild/compiler/discovery/_helpers/yml/entry_lines.py

@@ -0,0 +1,67 @@
+"""Index the outermost `- name:` entries of authored YAML declaration files."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterator, Mapping
+from functools import lru_cache
+
+_ENTRY_PREFIX: re.Pattern[str] = re.compile(r"(?P<indent>\s*)-\s*name:")
+_COMMENT_START: str = "#"
+_QUOTES: tuple[str, ...] = ("", "'", '"')
+
+
+@lru_cache(maxsize=16)
+def yaml_entry_lines(contents: str) -> Mapping[str, int]:
+    """Index every entry name once so per-resource lookups stay linear in file size."""
+
+    best: dict[str, tuple[int, int]] = {}
+    index: int
+    line: str
+    for index, line in enumerate(contents.splitlines(), start=1):
+        prefix: re.Match[str] | None = _ENTRY_PREFIX.match(line)
+        if prefix is None:
+            continue
+        indent: int = len(prefix.group("indent"))
+        for name in _entry_names(line[prefix.end() :]):
+            current: tuple[int, int] | None = best.get(name)
+            if current is None or indent < current[0]:
+                best[name] = (indent, index)
+    return {name: line_number for name, (_, line_number) in best.items()}
+
+
+def _entry_names(value: str) -> Iterator[str]:
+    """Yield every name `\\s*(['"]?)<name>\\1\\s*(#.*)?$` accepts for one entry value."""
+
+    closings: tuple[int, ...] = _closing_positions(value)
+    for start in range(_leading_whitespace(text=value, start=0) + 1):
+        for quote in _QUOTES:
+            if not value.startswith(quote, start):
+                continue
+            body_start: int = start + len(quote)
+            for closing in closings:
+                end: int = closing - len(quote)
+                if end >= body_start and value.startswith(quote, end):
+                    yield value[body_start:end]
+
+
+def _closing_positions(value: str) -> tuple[int, ...]:
+    """Return positions after which only whitespace and an optional comment remain."""
+
+    positions: list[int] = []
+    tail: int = len(value)
+    while tail >= 0:
+        position: int = tail
+        positions.append(position)
+        while position > 0 and value[position - 1].isspace():
+            position -= 1
+            positions.append(position)
+        tail = value.rfind(_COMMENT_START, 0, tail)
+    return tuple(sorted(set(positions)))
+
+
+def _leading_whitespace(*, text: str, start: int) -> int:
+    position: int = start
+    while position < len(text) and text[position].isspace():
+        position += 1
+    return position

src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py

@@ -2,23 +2,10 @@
 
 from __future__ import annotations
 
-import re
+from sqlbuild.compiler.discovery._helpers.yml.entry_lines import yaml_entry_lines
 
 
 def yaml_entry_line(*, contents: str, name: str) -> int | None:
     """Return the 1-based line of the outermost `- name: <name>` entry, if present."""
 
-    pattern: re.Pattern[str] = re.compile(
-        rf"^(?P<indent>\s*)-\s*name:\s*(?P<quote>['\"]?){re.escape(name)}(?P=quote)\s*(#.*)?$"
-    )
-    best: tuple[int, int] | None = None
-    index: int
-    line: str
-    for index, line in enumerate(contents.splitlines(), start=1):
-        match: re.Match[str] | None = pattern.match(line)
-        if match is None:
-            continue
-        indent: int = len(match.group("indent"))
-        if best is None or indent < best[0]:
-            best = (indent, index)
-    return None if best is None else best[1]
+    return yaml_entry_lines(contents).get(name)

src/sqlbuild/compiler/sql_analysis/_helpers/scanning.py

@@ -231,6 +231,50 @@ def iter_code_positions_impl(
         index += 1
 
 
+def skip_ignorable_impl(
+    *, sql: str, start: int, context: str = "SQL", syntax: SqlLexicalSyntax | None = None
+) -> int:
+    """Return the first offset at or after `start` that is not whitespace or a comment."""
+
+    index: int = start
+    while index < len(sql):
+        if sql[index].isspace():
+            index += 1
+            continue
+        if sql[index] in SQL_TEXT_START_CHARACTERS:
+            return index
+        comment_end: int | None = _non_code_end(
+            sql=sql, index=index, context=context, syntax=syntax
+        )
+        if comment_end is None:
+            return index
+        index = comment_end
+    return index
+
+
+def strip_ignorable_impl(
+    *, sql: str, context: str = "SQL", syntax: SqlLexicalSyntax | None = None
+) -> str:
+    """Return the SQL without leading or trailing whitespace and comments."""
+
+    code_start: int = skip_ignorable_impl(sql=sql, start=0, context=context, syntax=syntax)
+    code_end: int = code_start
+    index: int = code_start
+    while index < len(sql):
+        non_code_end: int | None = _non_code_end(
+            sql=sql, index=index, context=context, syntax=syntax
+        )
+        if non_code_end is None:
+            index += 1
+            if not sql[index - 1].isspace():
+                code_end = index
+            continue
+        if sql[index] in SQL_TEXT_START_CHARACTERS:
+            code_end = non_code_end
+        index = non_code_end
+    return sql[code_start:code_end]
+
+
 def _non_code_end(
     *, sql: str, index: int, context: str, syntax: SqlLexicalSyntax | None
 ) -> int | None: