fix(compile): linear missing-description locations and comment-tolerant expected fixtures

crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs

@@ -512,7 +512,7 @@ fn projection_name(expression: &str, file: &str, label: &str) -> Result<String,
     if let Some(position) = find_last_top_level_keyword(expression, "AS")? {
         let alias_start = skip_ignorable(expression, position + 2)?;
         if let Some((alias, end)) = read_identifier(expression, alias_start)
-            && expression[end..].trim().is_empty()
+            && skip_ignorable(expression, end)? == expression.len()
         {
             return Ok(alias);
         }
@@ -880,8 +880,34 @@ fn scan_code<T>(
     Ok(None)
 }
 
-fn non_empty_trimmed(value: &str) -> Option<&str> {
-    Some(value.trim()).filter(|trimmed| !trimmed.is_empty())
+/// Return `value` without surrounding whitespace and comments, or `None` when no code remains.
+fn non_empty_trimmed(value: &str) -> Result<Option<&str>, String> {
+    if !value.contains("--") && !value.contains("/*") {
+        return Ok(Some(value.trim()).filter(|trimmed| !trimmed.is_empty()));
+    }
+    let start = skip_ignorable(value, 0)?;
+    let mut end = start;
+    let mut index = start;
+    while index < value.len() {
+        if let Some(comment) = comment_end(value.as_bytes(), index)
+            .map_err(|error| scan_error_message(error, "SQL test"))?
+        {
+            index = comment;
+            continue;
+        }
+        let next = skip_non_code(value, index)?;
+        if next != index {
+            end = next;
+            index = next;
+            continue;
+        }
+        let width = char_len(value, index);
+        if !value[index..].starts_with(char::is_whitespace) {
+            end = index + width;
+        }
+        index += width;
+    }
+    Ok(Some(&value[start..end]).filter(|trimmed| !trimmed.is_empty()))
 }
 
 /// Split on top-level `UNION`, `INTERSECT` and `EXCEPT`, each with an optional `ALL`/`DISTINCT`.
@@ -895,7 +921,7 @@ pub(crate) fn split_set_operations(sql: &str) -> Result<Vec<&str>, String> {
         let Some(end) = set_operator_end(sql, index)? else {
             return Ok(CodeStep::Advance);
         };
-        values.extend(non_empty_trimmed(&sql[start..index]));
+        values.extend(non_empty_trimmed(&sql[start..index])?);
         let mut resume = skip_ignorable(sql, end)?;
         if let Some(quantifier_end) =
             consume_keyword(sql, resume, "ALL").or_else(|| consume_keyword(sql, resume, "DISTINCT"))
@@ -905,7 +931,7 @@ pub(crate) fn split_set_operations(sql: &str) -> Result<Vec<&str>, String> {
         start = resume;
         Ok(CodeStep::Resume(resume))
     })?;
-    values.extend(non_empty_trimmed(&sql[start..]));
+    values.extend(non_empty_trimmed(&sql[start..])?);
     Ok(values)
 }
 
@@ -927,12 +953,12 @@ pub(crate) fn split_top_level(sql: &str, separator: u8) -> Result<Vec<&str>, Str
     let mut start = 0;
     scan_code::<()>(sql, 0, |index, depth| {
         if depth == 0 && byte_at(sql, index) == Some(separator) {
-            values.extend(non_empty_trimmed(&sql[start..index]));
+            values.extend(non_empty_trimmed(&sql[start..index])?);
             start = index + 1;
         }
         Ok(CodeStep::Advance)
     })?;
-    values.extend(non_empty_trimmed(&sql[start..]));
+    values.extend(non_empty_trimmed(&sql[start..])?);
     Ok(values)
 }
 

crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_extraction.rs

@@ -91,12 +91,26 @@ fn given_expected_cte_sql_when_scanning_top_level_then_set_operations_commas_and
             ]),
             expected_commas: Ok(vec![
                 "SELECT 'x, UNION (' AS a",
-                "/* UNION , FROM */ \"b)\"",
+                "\"b)\"",
                 "`c,d`",
                 "f(1, (SELECT 2 FROM t UNION SELECT 3)) -- , FROM\nFROM t",
             ]),
             expected_from: Ok(Some(107)),
         },
+        TopLevelScanTestCase {
+            description: "comments around branches and projections are not code",
+            sql: "-- lead\nSELECT 1 AS a, /* b */ 2 AS b -- first\n/* x */ UNION ALL -- second\nSELECT '--' AS a, 3 AS b /* end */",
+            expected_unions: Ok(vec![
+                "SELECT 1 AS a, /* b */ 2 AS b",
+                "SELECT '--' AS a, 3 AS b",
+            ]),
+            expected_commas: Ok(vec![
+                "SELECT 1 AS a",
+                "2 AS b -- first\n/* x */ UNION ALL -- second\nSELECT '--' AS a",
+                "3 AS b",
+            ]),
+            expected_from: Ok(None),
+        },
         TopLevelScanTestCase {
             description: "keyword boundaries and empty pieces",
             sql: ",SELECT reunion, union_id,, x FROMAGE UNION",

scripts/cold_compile_performance/_helpers/diagnostic_project.py

@@ -132,3 +132,34 @@ def write_single_model_diagnostic_project(*, project_dir: Path, diagnostics: int
         + ",\n".join(projections)
         + '\nFROM __ref("orders") AS o\n'
     )
+
+
+def write_undescribed_project(*, project_dir: Path, model_count: int, width: int = 6) -> None:
+    """Generate models, seeds, and one large source file whose resources all lack descriptions."""
+    project_dir.mkdir(parents=True, exist_ok=True)
+    for folder in ("models", "sources", "seeds"):
+        (project_dir / folder).mkdir(exist_ok=True)
+    (project_dir / "sqlbuild_project.toml").write_text(
+        'name = "undescribed_orders"\nadapter = "duckdb"\n[rules]\nselect = []\n'
+    )
+    columns: list[str] = [f"amount_{index:03}" for index in range(width)]
+    sources: list[str] = ["sources:"]
+    seeds: list[str] = ["seeds:"]
+    for index in range(model_count):
+        sources.extend(
+            (f"  - name: orders_{index:05}", "    schema: main", f"    table: orders_{index:05}")
+        )
+        sources.append("    columns:")
+        sources.extend(f"      - name: {column}\n        type: INTEGER" for column in columns)
+        (project_dir / "models" / f"order_totals_{index:05}.sql").write_text(
+            "MODEL (materialized view);\n"
+            f'SELECT {", ".join(columns)} FROM __source("orders_{index:05}")\n'
+        )
+        if index % 10 == 0:
+            seeds.append(f"  - name: products_{index:05}\n    columns:")
+            seeds.extend(f"      - name: {column}\n        type: INTEGER" for column in columns)
+            (project_dir / "seeds" / f"products_{index:05}.csv").write_text(
+                ",".join(columns) + "\n" + ",".join("1" for _ in columns) + "\n"
+            )
+    (project_dir / "sources" / "orders.yml").write_text("\n".join(sources) + "\n")
+    (project_dir / "seeds" / "products.yml").write_text("\n".join(seeds) + "\n")

src/sqlbuild/compiler/compile/_helpers/analysis/tests.py

@@ -15,14 +15,19 @@
     split_set_operation_branches,
 )
 from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql
+from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax
 
 _POLYGLOT_VALUES_EXPRESSION_NAME: str = "Values"
 _POLYGLOT_VALUES_SET_ALIAS: str = "_values"
 _SCAN_CONTEXT: str = "SQL test"
 
 
 def extract_expected_branch_column_names_with_sql_analysis(
-    *, sql: str, file_label: str, label: str = "__expected__<model>"
+    *,
+    sql: str,
+    file_label: str,
+    syntax: SqlLexicalSyntax,
+    label: str = "__expected__<model>",
 ) -> tuple[tuple[str, ...], ...] | None:
     """Return expected SELECT branch names using required Polyglot analysis."""
 
@@ -32,17 +37,23 @@ def extract_expected_branch_column_names_with_sql_analysis(
         parsed_expression: Any = polyglot_module.parse_one(sql, dialect="generic")
     except polyglot_module.PolyglotError:
         return _extract_unparsed_branch_names(
-            sql=sql, polyglot_module=polyglot_module, file_label=file_label, label=label
+            sql=sql,
+            polyglot_module=polyglot_module,
+            file_label=file_label,
+            label=label,
+            syntax=syntax,
         )
     return _extract_branch_names(expression=parsed_expression, file_label=file_label, label=label)
 
 
 def _extract_unparsed_branch_names(
-    *, sql: str, polyglot_module: Any, file_label: str, label: str
+    *, sql: str, polyglot_module: Any, file_label: str, label: str, syntax: SqlLexicalSyntax
 ) -> tuple[tuple[str, ...], ...] | None:
     """Parse each top-level set-operation branch separately when the whole query cannot parse."""
 
-    branches: tuple[str, ...] = split_set_operation_branches(sql=sql, context=_SCAN_CONTEXT)
+    branches: tuple[str, ...] = split_set_operation_branches(
+        sql=sql, context=_SCAN_CONTEXT, syntax=syntax
+    )
     if len(branches) <= 1:
         return None
     try:

src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py

@@ -762,12 +762,14 @@ def _extract_expected_branch_column_names(
 ) -> tuple[tuple[str, ...], ...]:
     sql_analysis_column_names: tuple[tuple[str, ...], ...] | None = (
         extract_expected_branch_column_names_with_sql_analysis(
-            sql=sql, file_label=file_label, label=label
+            sql=sql, file_label=file_label, label=label, syntax=syntax
         )
     )
     if sql_analysis_column_names is not None:
         return sql_analysis_column_names
-    branches: tuple[str, ...] = split_set_operation_branches(sql=sql, context=_CONTEXT)
+    branches: tuple[str, ...] = split_set_operation_branches(
+        sql=sql, context=_CONTEXT, syntax=syntax
+    )
     return tuple(
         _extract_expected_select_column_names(
             branch_sql=branch, file_label=file_label, label=label, syntax=syntax
@@ -837,9 +839,13 @@ def _extract_expected_projection_name(
     alias_name: str | None = _extract_as_alias(expression=expression, syntax=syntax)
     if alias_name is not None:
         return alias_name
-    stripped_expression: str = expression.strip()
-    if _is_simple_identifier(stripped_expression):
-        return stripped_expression
+    start: int = _skip_ignorable(sql=expression, start=0, syntax=syntax)
+    if start < len(expression) and is_identifier_start(expression[start]):
+        name: str
+        end: int
+        name, end = _read_identifier(sql=expression, start=start, file_label="projection")
+        if _skip_ignorable(sql=expression, start=end, syntax=syntax) == len(expression):
+            return name
     raise CompileInputError(
         f"SQL test '{file_label}' must alias every non-trivial {label} projection"
     )
@@ -862,17 +868,11 @@ def _extract_as_alias(*, expression: str, syntax: SqlLexicalSyntax) -> str | Non
                 start=alias_index,
                 file_label="projection",
             )
-            if not expression[alias_end:].strip():
+            if _skip_ignorable(sql=expression, start=alias_end, syntax=syntax) == len(expression):
                 last_alias_name = alias_name
     return last_alias_name
 
 
-def _is_simple_identifier(value: str) -> bool:
-    if not value or not is_identifier_start(value[0]):
-        return False
-    return all(is_identifier_character(character) for character in value[1:])
-
-
 def _contains_select_star(*, sql: str, syntax: SqlLexicalSyntax) -> bool:
     index: int
     for index, _depth in iter_code_positions(sql=sql, context=_CONTEXT, syntax=syntax):

src/sqlbuild/compiler/compile/constants.py

@@ -96,7 +96,7 @@
 )
 POLYGLOT_SELECT_EXPRESSION_NAME: str = "Select"
 POLYGLOT_COLUMN_EXPRESSION_NAME: str = "Column"
-POLYGLOT_WRAPPER_EXPRESSION_NAMES: frozenset[str] = frozenset({"Subquery", "Paren"})
+POLYGLOT_WRAPPER_EXPRESSION_NAMES: frozenset[str] = frozenset({"Annotated", "Subquery", "Paren"})
 
 TABLE_FUNCTION_RETURN_KEYS: frozenset[str] = frozenset({"table"})
 

src/sqlbuild/compiler/discovery/_helpers/yml/entry_lines.py

@@ -0,0 +1,67 @@
+"""Index the outermost `- name:` entries of authored YAML declaration files."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterator, Mapping
+from functools import lru_cache
+
+_ENTRY_PREFIX: re.Pattern[str] = re.compile(r"(?P<indent>\s*)-\s*name:")
+_COMMENT_START: str = "#"
+_QUOTES: tuple[str, ...] = ("", "'", '"')
+
+
+@lru_cache(maxsize=16)
+def yaml_entry_lines(contents: str) -> Mapping[str, int]:
+    """Index every entry name once so per-resource lookups stay linear in file size."""
+
+    best: dict[str, tuple[int, int]] = {}
+    index: int
+    line: str
+    for index, line in enumerate(contents.splitlines(), start=1):
+        prefix: re.Match[str] | None = _ENTRY_PREFIX.match(line)
+        if prefix is None:
+            continue
+        indent: int = len(prefix.group("indent"))
+        for name in _entry_names(line[prefix.end() :]):
+            current: tuple[int, int] | None = best.get(name)
+            if current is None or indent < current[0]:
+                best[name] = (indent, index)
+    return {name: line_number for name, (_, line_number) in best.items()}
+
+
+def _entry_names(value: str) -> Iterator[str]:
+    """Yield every name `\\s*(['"]?)<name>\\1\\s*(#.*)?$` accepts for one entry value."""
+
+    closings: tuple[int, ...] = _closing_positions(value)
+    for start in range(_leading_whitespace(text=value, start=0) + 1):
+        for quote in _QUOTES:
+            if not value.startswith(quote, start):
+                continue
+            body_start: int = start + len(quote)
+            for closing in closings:
+                end: int = closing - len(quote)
+                if end >= body_start and value.startswith(quote, end):
+                    yield value[body_start:end]
+
+
+def _closing_positions(value: str) -> tuple[int, ...]:
+    """Return positions after which only whitespace and an optional comment remain."""
+
+    positions: list[int] = []
+    tail: int = len(value)
+    while tail >= 0:
+        position: int = tail
+        positions.append(position)
+        while position > 0 and value[position - 1].isspace():
+            position -= 1
+            positions.append(position)
+        tail = value.rfind(_COMMENT_START, 0, tail)
+    return tuple(sorted(set(positions)))
+
+
+def _leading_whitespace(*, text: str, start: int) -> int:
+    position: int = start
+    while position < len(text) and text[position].isspace():
+        position += 1
+    return position

src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py

@@ -2,23 +2,10 @@
 
 from __future__ import annotations
 
-import re
+from sqlbuild.compiler.discovery._helpers.yml.entry_lines import yaml_entry_lines
 
 
 def yaml_entry_line(*, contents: str, name: str) -> int | None:
     """Return the 1-based line of the outermost `- name: <name>` entry, if present."""
 
-    pattern: re.Pattern[str] = re.compile(
-        rf"^(?P<indent>\s*)-\s*name:\s*(?P<quote>['\"]?){re.escape(name)}(?P=quote)\s*(#.*)?$"
-    )
-    best: tuple[int, int] | None = None
-    index: int
-    line: str
-    for index, line in enumerate(contents.splitlines(), start=1):
-        match: re.Match[str] | None = pattern.match(line)
-        if match is None:
-            continue
-        indent: int = len(match.group("indent"))
-        if best is None or indent < best[0]:
-            best = (indent, index)
-    return None if best is None else best[1]
+    return yaml_entry_lines(contents).get(name)

src/sqlbuild/compiler/sql_analysis/_helpers/scanning.py

@@ -231,6 +231,50 @@ def iter_code_positions_impl(
         index += 1
 
 
+def skip_ignorable_impl(
+    *, sql: str, start: int, context: str = "SQL", syntax: SqlLexicalSyntax | None = None
+) -> int:
+    """Return the first offset at or after `start` that is not whitespace or a comment."""
+
+    index: int = start
+    while index < len(sql):
+        if sql[index].isspace():
+            index += 1
+            continue
+        if sql[index] in SQL_TEXT_START_CHARACTERS:
+            return index
+        comment_end: int | None = _non_code_end(
+            sql=sql, index=index, context=context, syntax=syntax
+        )
+        if comment_end is None:
+            return index
+        index = comment_end
+    return index
+
+
+def strip_ignorable_impl(
+    *, sql: str, context: str = "SQL", syntax: SqlLexicalSyntax | None = None
+) -> str:
+    """Return the SQL without leading or trailing whitespace and comments."""
+
+    code_start: int = skip_ignorable_impl(sql=sql, start=0, context=context, syntax=syntax)
+    code_end: int = code_start
+    index: int = code_start
+    while index < len(sql):
+        non_code_end: int | None = _non_code_end(
+            sql=sql, index=index, context=context, syntax=syntax
+        )
+        if non_code_end is None:
+            index += 1
+            if not sql[index - 1].isspace():
+                code_end = index
+            continue
+        if sql[index] in SQL_TEXT_START_CHARACTERS:
+            code_end = non_code_end
+        index = non_code_end
+    return sql[code_start:code_end]
+
+
 def _non_code_end(
     *, sql: str, index: int, context: str, syntax: SqlLexicalSyntax | None
 ) -> int | None: