perf(compile): index YAML entry lines once for missing-description locations

Locating each undescribed seed or source rescanned its whole YAML file with
a per-name regex, so error-heavy compiles grew quadratically with entries per
file. Index every entry name once per file and look names up from that index,
preserving the outermost-entry and pattern semantics exactly.

Author: kvlonge

scripts/cold_compile_performance/_helpers/diagnostic_project.py

@@ -132,3 +132,34 @@ def write_single_model_diagnostic_project(*, project_dir: Path, diagnostics: int
         + ",\n".join(projections)
         + '\nFROM __ref("orders") AS o\n'
     )
+
+
+def write_undescribed_project(*, project_dir: Path, model_count: int, width: int = 6) -> None:
+    """Generate models, seeds, and one large source file whose resources all lack descriptions."""
+    project_dir.mkdir(parents=True, exist_ok=True)
+    for folder in ("models", "sources", "seeds"):
+        (project_dir / folder).mkdir(exist_ok=True)
+    (project_dir / "sqlbuild_project.toml").write_text(
+        'name = "undescribed_orders"\nadapter = "duckdb"\n[rules]\nselect = []\n'
+    )
+    columns: list[str] = [f"amount_{index:03}" for index in range(width)]
+    sources: list[str] = ["sources:"]
+    seeds: list[str] = ["seeds:"]
+    for index in range(model_count):
+        sources.extend(
+            (f"  - name: orders_{index:05}", "    schema: main", f"    table: orders_{index:05}")
+        )
+        sources.append("    columns:")
+        sources.extend(f"      - name: {column}\n        type: INTEGER" for column in columns)
+        (project_dir / "models" / f"order_totals_{index:05}.sql").write_text(
+            "MODEL (materialized view);\n"
+            f'SELECT {", ".join(columns)} FROM __source("orders_{index:05}")\n'
+        )
+        if index % 10 == 0:
+            seeds.append(f"  - name: products_{index:05}\n    columns:")
+            seeds.extend(f"      - name: {column}\n        type: INTEGER" for column in columns)
+            (project_dir / "seeds" / f"products_{index:05}.csv").write_text(
+                ",".join(columns) + "\n" + ",".join("1" for _ in columns) + "\n"
+            )
+    (project_dir / "sources" / "orders.yml").write_text("\n".join(sources) + "\n")
+    (project_dir / "seeds" / "products.yml").write_text("\n".join(seeds) + "\n")

src/sqlbuild/compiler/discovery/_helpers/yml/entry_lines.py

@@ -0,0 +1,67 @@
+"""Index the outermost `- name:` entries of authored YAML declaration files."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterator, Mapping
+from functools import lru_cache
+
+_ENTRY_PREFIX: re.Pattern[str] = re.compile(r"(?P<indent>\s*)-\s*name:")
+_COMMENT_START: str = "#"
+_QUOTES: tuple[str, ...] = ("", "'", '"')
+
+
+@lru_cache(maxsize=16)
+def yaml_entry_lines(contents: str) -> Mapping[str, int]:
+    """Index every entry name once so per-resource lookups stay linear in file size."""
+
+    best: dict[str, tuple[int, int]] = {}
+    index: int
+    line: str
+    for index, line in enumerate(contents.splitlines(), start=1):
+        prefix: re.Match[str] | None = _ENTRY_PREFIX.match(line)
+        if prefix is None:
+            continue
+        indent: int = len(prefix.group("indent"))
+        for name in _entry_names(line[prefix.end() :]):
+            current: tuple[int, int] | None = best.get(name)
+            if current is None or indent < current[0]:
+                best[name] = (indent, index)
+    return {name: line_number for name, (_, line_number) in best.items()}
+
+
+def _entry_names(value: str) -> Iterator[str]:
+    """Yield every name `\\s*(['"]?)<name>\\1\\s*(#.*)?$` accepts for one entry value."""
+
+    closings: tuple[int, ...] = _closing_positions(value)
+    for start in range(_leading_whitespace(text=value, start=0) + 1):
+        for quote in _QUOTES:
+            if not value.startswith(quote, start):
+                continue
+            body_start: int = start + len(quote)
+            for closing in closings:
+                end: int = closing - len(quote)
+                if end >= body_start and value.startswith(quote, end):
+                    yield value[body_start:end]
+
+
+def _closing_positions(value: str) -> tuple[int, ...]:
+    """Return positions after which only whitespace and an optional comment remain."""
+
+    positions: list[int] = []
+    tail: int = len(value)
+    while tail >= 0:
+        position: int = tail
+        positions.append(position)
+        while position > 0 and value[position - 1].isspace():
+            position -= 1
+            positions.append(position)
+        tail = value.rfind(_COMMENT_START, 0, tail)
+    return tuple(sorted(set(positions)))
+
+
+def _leading_whitespace(*, text: str, start: int) -> int:
+    position: int = start
+    while position < len(text) and text[position].isspace():
+        position += 1
+    return position

src/sqlbuild/compiler/discovery/main/_yaml_entry_line.py

@@ -2,23 +2,10 @@
 
 from __future__ import annotations
 
-import re
+from sqlbuild.compiler.discovery._helpers.yml.entry_lines import yaml_entry_lines
 
 
 def yaml_entry_line(*, contents: str, name: str) -> int | None:
     """Return the 1-based line of the outermost `- name: <name>` entry, if present."""
 
-    pattern: re.Pattern[str] = re.compile(
-        rf"^(?P<indent>\s*)-\s*name:\s*(?P<quote>['\"]?){re.escape(name)}(?P=quote)\s*(#.*)?$"
-    )
-    best: tuple[int, int] | None = None
-    index: int
-    line: str
-    for index, line in enumerate(contents.splitlines(), start=1):
-        match: re.Match[str] | None = pattern.match(line)
-        if match is None:
-            continue
-        indent: int = len(match.group("indent"))
-        if best is None or indent < best[0]:
-            best = (indent, index)
-    return None if best is None else best[1]
+    return yaml_entry_lines(contents).get(name)

tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py

@@ -517,6 +517,14 @@ class RequiredDescriptionAggregateCase:
     expected_kinds: frozenset[str]
 
 
+@dataclass(frozen=True)
+class RequiredDescriptionExactOutputCase:
+    description: str
+    files: tuple[tuple[str, str], ...]
+    expected_diagnostics: tuple[tuple[str, str, str, int, int], ...]
+    expected_help: tuple[str, ...]
+
+
 @dataclass(frozen=True)
 class RequiredDescriptionPlanCase:
     description: str

tests/e2e/src/sqlbuild/cli/commands/main/compile/test_diagnostic_compile_performance.py

@@ -18,6 +18,7 @@
     write_diagnostic_project,
     write_function_name_project,
     write_single_model_diagnostic_project,
+    write_undescribed_project,
 )
 from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import DiagnosticPerformanceCase
 
@@ -102,6 +103,40 @@ def test_given_one_model_with_many_diagnostics_when_compiling_then_scales_linear
     assert timings[1] <= timings[0] * 4.5 + 1.0
 
 
+@pytest.mark.parametrize(
+    "test_case",
+    [
+        DiagnosticPerformanceCase(
+            "many_undescribed_resources",
+            diagnostic_count=500,
+            expected_max_wall_seconds=15.0,
+        )
+    ],
+    ids=lambda case: case.description,
+)
+def test_given_many_undescribed_resources_when_compiling_then_scales_linearly(
+    test_case: DiagnosticPerformanceCase, tmp_path: Path
+) -> None:
+    timings: list[float] = []
+    for multiplier in (1, 4):
+        project: Path = tmp_path / f"undescribed_{multiplier}"
+        count: int = test_case.diagnostic_count * multiplier
+        write_undescribed_project(project_dir=project, model_count=count)
+        elapsed, payload = measure_diagnostic_compile(
+            project_dir=project, timeout=test_case.expected_timeout_seconds
+        )
+        kinds: Counter[tuple[str, str]] = Counter(
+            (item["code"], item["resource_type"]) for item in payload["diagnostics"]
+        )
+        assert kinds == Counter(
+            {("P010", "model"): count, ("P010", "source"): count, ("P010", "seed"): count // 10}
+        )
+        timings.append(elapsed)
+    _LOGGER.info("undescribed resources 1x/4x wall=%s", timings)
+    assert max(timings) < test_case.expected_max_wall_seconds
+    assert timings[1] <= timings[0] * 5.0 + 1.0
+
+
 @pytest.mark.parametrize(
     "test_case",
     [

tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py

@@ -17,6 +17,7 @@
     PathDefaultDescriptionCase,
     RequiredDescriptionAggregateCase,
     RequiredDescriptionCase,
+    RequiredDescriptionExactOutputCase,
     RequiredDescriptionPlanCase,
 )
 from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import (
@@ -43,6 +44,27 @@
 )
 
 
+_SHADOWED_SOURCES_YAML: str = (
+    "sources:\n"
+    "  - name: raw_orders\n"
+    "    schema: main\n"
+    "    table: orders\n"
+    "    columns:\n"
+    "      - name: raw_refunds\n"
+    "        type: INTEGER\n"
+    '  - name: "raw_refunds"  # refunds feed\n'
+    "    schema: main\n"
+    "    table: refunds\n"
+    "  - name: 'raw_customers'\n"
+    "    schema: main\n"
+    "    table: customers\n"
+)
+_TWO_SEEDS_YAML: str = (
+    "seeds:\n"
+    "  - name: product_types\n" + _SEED_COLUMNS + "  - name: product_labels\n" + _SEED_COLUMNS
+)
+
+
 _PROVIDER_DESCRIBED: str = (
     "from sqlbuild.providers import Provider\n\n\n"
     "class OrdersApi(Provider):\n"
@@ -534,6 +556,89 @@ def test_given_every_resource_kind_undescribed_when_compiling_then_one_run_repor
     assert kinds == test_case.expected_kinds
 
 
+@pytest.mark.parametrize(
+    "test_case",
+    [
+        RequiredDescriptionExactOutputCase(
+            description="seed and source entries located at their outermost declarations",
+            files=(
+                ("models/order_summary.sql", "MODEL ();\nSELECT 1 AS order_count\n"),
+                ("seeds/product_types.csv", _SEED_CSV),
+                ("seeds/product_labels.csv", _SEED_CSV),
+                ("seeds/products.yml", _TWO_SEEDS_YAML),
+                ("sources/raw.yml", _SHADOWED_SOURCES_YAML),
+            ),
+            expected_diagnostics=(
+                (
+                    "P010",
+                    "model 'order_summary' has no description",
+                    "models/order_summary.sql",
+                    1,
+                    1,
+                ),
+                ("P010", "seed 'product_types' has no description", "seeds/products.yml", 2, 1),
+                ("P010", "seed 'product_labels' has no description", "seeds/products.yml", 8, 1),
+                ("P010", "source 'raw_orders' has no description", "sources/raw.yml", 2, 1),
+                ("P010", "source 'raw_refunds' has no description", "sources/raw.yml", 8, 1),
+                ("P010", "source 'raw_customers' has no description", "sources/raw.yml", 11, 1),
+            ),
+            expected_help=(
+                "to describe model 'order_summary', add this to the MODEL header in "
+                "models/order_summary.sql:\n"
+                "            MODEL (\n"
+                '              description "What one row of order_summary represents",\n'
+                "              ...\n"
+                "            );",
+                (
+                    "to describe seed 'product_types', add `description` to its entry in seeds/products.yml:\n"
+                    "            - name: product_types\n"
+                    "              description: What one row of product_types represents"
+                ),
+                (
+                    "to describe seed 'product_labels', add `description` to its entry in seeds/products.yml:\n"
+                    "            - name: product_labels\n"
+                    "              description: What one row of product_labels represents"
+                ),
+                (
+                    "to describe source 'raw_orders', add `description` to its entry in sources/raw.yml:\n"
+                    "            - name: raw_orders\n"
+                    "              description: What one row of raw_orders represents"
+                ),
+                (
+                    "to describe source 'raw_refunds', add `description` to its entry in sources/raw.yml:\n"
+                    "            - name: raw_refunds\n"
+                    "              description: What one row of raw_refunds represents"
+                ),
+                (
+                    "to describe source 'raw_customers', add `description` to its entry in sources/raw.yml:\n"
+                    "            - name: raw_customers\n"
+                    "              description: What one row of raw_customers represents"
+                ),
+            ),
+        ),
+    ],
+    ids=lambda case: case.description,
+)
+def test_given_undescribed_yaml_entries_when_compiling_then_reports_exact_ordered_diagnostics(
+    test_case: RequiredDescriptionExactOutputCase, tmp_path: Path
+) -> None:
+    result: subprocess.CompletedProcess[str] = compile_inline_files(
+        tmp_path=tmp_path, files=test_case.files
+    )
+
+    payload: dict[str, Any] = json.loads(result.stdout)
+    diagnostics: list[dict[str, Any]] = payload["diagnostics"]
+    assert result.returncode == 1, result.stdout + result.stderr
+    assert (
+        tuple(
+            (item["code"], item["message"], item["path"], item["line"], item["column"])
+            for item in diagnostics
+        )
+        == test_case.expected_diagnostics
+    )
+    assert tuple(item["help"] for item in diagnostics) == test_case.expected_help
+
+
 @pytest.mark.parametrize(
     "test_case",
     [

tests/unit/src/sqlbuild/compiler/discovery/main/_test_types.py

@@ -8,3 +8,11 @@ class SerializeHookEntriesTestCase:
     sql_fields: tuple[str, ...]
     python_hook_fields: dict[str, dict[str, object]]
     expected_hooks: list[dict[str, object]]
+
+
+@dataclass(frozen=True)
+class YamlEntryLineTestCase:
+    description: str
+    contents: str
+    name: str
+    expected_line: int | None

tests/unit/src/sqlbuild/compiler/discovery/main/test_yaml_entry_line.py

@@ -0,0 +1,49 @@
+"""Unit tests for locating named entries in authored YAML declaration files."""
+
+from __future__ import annotations
+
+import pytest
+
+from sqlbuild.compiler.discovery.main._yaml_entry_line import yaml_entry_line
+from tests.unit.src.sqlbuild.compiler.discovery.main._test_types import YamlEntryLineTestCase
+
+_SOURCES: str = (
+    "sources:\n"
+    "  - name: raw_orders\n"
+    "    columns:\n"
+    "      - name: raw_refunds\n"
+    '  - name: "raw_refunds"  # refunds feed\n'
+    "  - name: 'raw_customers'\n"
+    "  -   name:raw_products # products\n"
+    "  - name: raw_orders\n"
+)
+
+
+@pytest.mark.parametrize(
+    "test_case",
+    (
+        YamlEntryLineTestCase("plain entry", _SOURCES, "raw_orders", 2),
+        YamlEntryLineTestCase(
+            "outermost wins over earlier nested column", _SOURCES, "raw_refunds", 5
+        ),
+        YamlEntryLineTestCase("single quoted entry", _SOURCES, "raw_customers", 6),
+        YamlEntryLineTestCase("compact entry with trailing comment", _SOURCES, "raw_products", 7),
+        YamlEntryLineTestCase("comment text is not a name", _SOURCES, "products", None),
+        YamlEntryLineTestCase(
+            "unquoted line also matches its comment", _SOURCES, "raw_products # products", 7
+        ),
+        YamlEntryLineTestCase("missing entry", _SOURCES, "raw_shipments", None),
+        YamlEntryLineTestCase("mismatched quotes", "- name: \"raw_orders'\n", "raw_orders", None),
+    ),
+    ids=lambda case: case.description,
+)
+def test_given_yaml_declarations_when_locating_entry_then_returns_outermost_line(
+    test_case: YamlEntryLineTestCase,
+) -> None:
+    assert yaml_entry_line(contents=test_case.contents, name=test_case.name) == (
+        test_case.expected_line
+    )
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-vv"])