feat(compile)!: require a description on every named resource

.github/workflows/verify.yml

@@ -460,6 +460,7 @@ jobs:
             --kind "${KIND}" \
             --max-ratio "${MAX_RATIO}" \
             --models 3000 \
+            --base-root base \
             --base-python base/.venv/bin/python \
             --head-python .venv/bin/python
 

README.md

@@ -143,6 +143,7 @@ A model is a SQL file with a `MODEL()` header and a `SELECT`:
 
 ```sql
 MODEL (
+  description "One row per order",
   materialized table,
   columns (
     order_id (audits [not_null, unique]),

crates/sqlbuild-rules-native/src/engine/tests/test_sql_test_rules.rs

@@ -191,32 +191,6 @@ fn given_sql_test_facts_when_evaluating_project_rules_then_returns_expected_faul
                 "tests/unit/test_orders__paid.sql",
             ],
         },
-        test_types::SqlTestRulesTestCase {
-            description: "scenario descriptions reject generic numbering",
-            code: "SQBRTEST105",
-            tests: json!([]),
-            scenarios: json!([
-                {
-                    "source_path": "tests/scenarios/orders__paid.sql",
-                    "ownership_root": "tests/scenarios", "name": "orders__paid",
-                    "description": "Paid orders remain visible", "expected_model_names": [],
-                    "assertion_names": [], "assertion_target_model_names": [],
-                    "target_model_names": []
-                },
-                {
-                    "source_path": "tests/scenarios/orders__case_1.sql",
-                    "ownership_root": "tests/scenarios", "name": "orders__case_1",
-                    "description": "case 1", "expected_model_names": [],
-                    "assertion_names": [], "assertion_target_model_names": [],
-                    "target_model_names": []
-                }
-            ]),
-            scope_index: helpers::scope_index(),
-            config: json!({}),
-            expected_fault_count: 1,
-            expected_evaluated_models: 0,
-            expected_paths: &["tests/scenarios/orders__case_1.sql"],
-        },
         test_types::SqlTestRulesTestCase {
             description: "path-scoped ignore suppresses a test rule finding",
             code: "SQBRTEST104",

crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs

@@ -38,7 +38,6 @@ macro_rules! rule {
                     | "SQBRTEST102"
                     | "SQBRTEST103"
                     | "SQBRTEST104"
-                    | "SQBRTEST105"
                     | "SQBRTEST203"
                     | "SQBRTEST301"
             ),
@@ -294,13 +293,6 @@ pub(crate) fn catalogue() -> Vec<RuleMetadata> {
             "every SQL unit-test block must have a target-aware subject__expected_behavior name",
             "Add name \"<resolved_subject>__<expected_behavior>\" to this TEST header.",
         ),
-        rule!(
-            "SQBRTEST105",
-            "tests",
-            "scenario-business-description",
-            "scenario descriptions must identify a concrete business behavior",
-            "Write a non-generic SCENARIO description that states the business behavior under test.",
-        ),
         rule!(
             "SQBRTEST201",
             "tests",

crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs

@@ -90,9 +90,6 @@ pub(crate) fn evaluate_project(
     if let Some(rule) = evaluation.selected.get("SQBRTEST104") {
         faults.extend(structured_names(rule, &evaluation, &tests));
     }
-    if let Some(rule) = evaluation.selected.get("SQBRTEST105") {
-        faults.extend(scenario_descriptions(rule, &scenarios));
-    }
     if let Some(rule) = evaluation.selected.get(EMPTY_INPUT_RULE_CODE) {
         faults.extend(empty_input_only_faults(
             rule,
@@ -270,27 +267,6 @@ fn structured_names(
     faults
 }
 
-fn scenario_descriptions(rule: &RuleMetadata, scenarios: &[&SqlScenarioFact]) -> Vec<Fault> {
-    let mut faults: Vec<Fault> = Vec::new();
-    for scenario in scenarios {
-        let description = scenario.description.as_deref().unwrap_or("").trim();
-        let behavior = file_stem(&scenario.source_path)
-            .split_once("__")
-            .map(|(_, value)| value)
-            .unwrap_or("");
-        if description.is_empty() || generic(description) || generic(behavior) {
-            faults.push(path_fault(
-                rule,
-                &scenario.source_path,
-                "scenario description or filename uses a generic case label".into(),
-                "Write a concrete business description and a <subject>__<behavior>.sql filename."
-                    .into(),
-            ));
-        }
-    }
-    faults
-}
-
 fn expected_parent(
     evaluation: &ProjectEvaluationRequest<'_>,
     test: &SqlTestFact,

scripts/cold_compile_performance/_helpers/dense_project.py

@@ -156,6 +156,7 @@ def _write_sources(*, project_dir: Path, source_count: int) -> None:
         entries.extend(
             (
                 f"  - name: orders_{index:05d}",
+                f"    description: Generated source orders_{index:05d}.",
                 "    contract: enforced",
                 f'    expression: "(SELECT 1 AS id, CAST(1 AS DOUBLE) AS amount{extra})"',
                 "    columns:",
@@ -248,9 +249,8 @@ def _model_sql(*, index: int, source_count: int, function_count: int, macro_coun
             f'description "Order quantity measure"{audit})'
         )
     header: str = (
-        "MODEL (materialized table, contract enforced, columns (\n"
-        + ",\n".join(declarations)
-        + "\n));\n"
+        'MODEL (description "Generated dense model.", materialized table, contract enforced, '
+        "columns (\n" + ",\n".join(declarations) + "\n));\n"
     )
     source_index: int = _source_index(index=index, source_count=source_count)
     relation: str = (

scripts/cold_compile_performance/_helpers/diagnostic_project.py

@@ -16,7 +16,8 @@ def write_diagnostic_project(
     columns: list[str] = [f"amount_{index:03}" for index in range(width)]
     declarations: str = ", ".join(f"{column} (type INTEGER)" for column in columns)
     (models / "orders.sql").write_text(
-        f"MODEL (materialized view, contract enforced, columns ({declarations}));\nSELECT "
+        "MODEL (description 'Test model orders.', "
+        f"materialized view, contract enforced, columns ({declarations}));\nSELECT "
         + ", ".join(f"CAST(1 AS INTEGER) AS {column}" for column in columns)
     )
     for model_index in range(model_count):
@@ -31,12 +32,14 @@ def write_diagnostic_project(
         conversions: list[str] = [
             f"{columns[index % width]} = 'shipped' AS risky_{index:03}" for index in range(50)
         ]
-        sql: str = "MODEL (materialized view);\nWITH\n" + ",\n".join(ctes)
+        sql: str = "MODEL (description 'Test model.', materialized view);\nWITH\n" + ",\n".join(
+            ctes
+        )
         sql += "\nSELECT\n" + ",\n".join((*errors, *conversions)) + f"\nFROM {previous}\n"
         name = f"large_orders_{model_index}"
         (models / f"{name}.sql").write_text(sql)
         (models / f"downstream_orders_{model_index}.sql").write_text(
-            "MODEL (materialized view);\nSELECT "
+            "MODEL (description 'Test model.', materialized view);\nSELECT "
             + ", ".join(f"broken_{index:03} + 1 AS result_{index:03}" for index in range(50))
             + f' FROM __ref("{name}")'
         )
@@ -52,9 +55,13 @@ def write_cascade_project(*, project_dir: Path, depth: int, width: int = 50) ->
     )
     columns: list[str] = [f"amount_{index:03}" for index in range(width)]
     declarations: str = ", ".join(f"{column} (type INTEGER)" for column in columns)
-    header: str = f"MODEL (materialized view, contract enforced, columns ({declarations}));\n"
+    header: str = (
+        "MODEL (description 'Test model.', "
+        f"materialized view, contract enforced, columns ({declarations}));\n"
+    )
     (models / "input_orders.sql").write_text(
-        f"MODEL (materialized view, contract enforced, columns ({declarations}, "
+        "MODEL (description 'Test model input_orders.', "
+        f"materialized view, contract enforced, columns ({declarations}, "
         "ordered_at (type TIMESTAMP)));\nSELECT "
         + ", ".join(f"CAST(1 AS INTEGER) AS {column}" for column in columns)
         + ", CAST('2026-04-01' AS TIMESTAMP) AS ordered_at"
@@ -94,7 +101,7 @@ def write_function_name_project(*, project_dir: Path, model_count: int, calls: i
                 )
             )
         (models / f"wide_products_{model_index}.sql").write_text(
-            "MODEL (database warehouse, schema analytics);\nSELECT\n"
+            "MODEL (description 'Test model.', database warehouse, schema analytics);\nSELECT\n"
             + ",\n".join(projections)
             + "\nFROM (SELECT 'anvil' AS sku) AS products\nWHERE NOT (sku IN ('a', 'b'))\n"
         )
@@ -109,7 +116,8 @@ def write_single_model_diagnostic_project(*, project_dir: Path, diagnostics: int
         'name = "dense_diagnostic_orders"\nadapter = "duckdb"\n[rules]\nselect = []\n'
     )
     (models / "orders.sql").write_text(
-        "MODEL (materialized view, contract enforced, columns (order_id (type INTEGER), "
+        "MODEL (description 'Test model orders.', "
+        "materialized view, contract enforced, columns (order_id (type INTEGER), "
         "status (type VARCHAR)));\nSELECT CAST(1 AS INTEGER) AS order_id, "
         "CAST('open' AS VARCHAR) AS status"
     )
@@ -120,7 +128,7 @@ def write_single_model_diagnostic_project(*, project_dir: Path, diagnostics: int
         for index in range(diagnostics)
     ]
     (models / "order_diagnostics.sql").write_text(
-        "MODEL (materialized view);\nSELECT\n"
+        "MODEL (description 'Test model order_diagnostics.', materialized view);\nSELECT\n"
         + ",\n".join(projections)
         + '\nFROM __ref("orders") AS o\n'
     )

scripts/compile_performance_ratio/_helpers/measure.py

@@ -1,4 +1,4 @@
-"""Generate a benchmark project once and time alternating base and head compiles."""
+"""Generate benchmark projects and time alternating base and head compiles."""
 
 from __future__ import annotations
 
@@ -13,9 +13,11 @@
 
 from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project
 from scripts.compile_performance_ratio.constants import (
+    BASE_GENERATOR_ENTRY,
     BASE_LABEL,
     COMPILE_ENTRY,
     DENSE_KIND,
+    ERROR_TAIL_CHARACTERS,
     EXCLUDED_ENVIRONMENT_KEYS,
     EXCLUDED_ENVIRONMENT_PREFIX,
     FRESH_AUDIT_SHARE,
@@ -25,6 +27,7 @@
     FRESH_SOURCE_SHARE,
     FRESH_TEST_SHARE,
     HEAD_LABEL,
+    PYTHONPATH_KEY,
     REPORTED_PHASES,
 )
 from scripts.compile_performance_ratio.exceptions import CompileComparisonError
@@ -52,23 +55,47 @@ def write_benchmark_project(*, kind: str, project_dir: Path, models: int) -> Non
     )
 
 
+def write_base_benchmark_project(
+    *, base_root: Path, python: Path, kind: str, project_dir: Path, models: int
+) -> None:
+    """Write the base project with the base checkout's own generator, run by `python`."""
+
+    completed: subprocess.CompletedProcess[str] = subprocess.run(
+        [str(python), "-c", BASE_GENERATOR_ENTRY, kind, str(project_dir), str(models)],
+        cwd=base_root,
+        capture_output=True,
+        text=True,
+        env={**_compile_environment(), PYTHONPATH_KEY: str(base_root)},
+        check=False,
+    )
+    if completed.returncode != 0:
+        raise CompileComparisonError(
+            f"base project generation in {base_root} failed with exit {completed.returncode}: "
+            f"{completed.stderr[-ERROR_TAIL_CHARACTERS:]}"
+        )
+
+
 def compare_builds(
     *,
     kind: str,
     models: int,
-    project_dir: Path,
+    base_project_dir: Path,
+    head_project_dir: Path,
     base_python: Path,
     head_python: Path,
     runs: int,
 ) -> CompileComparison:
     """Alternate base and head cold compiles after one untimed warm-up compile per build."""
 
-    builds: tuple[tuple[str, Path], ...] = ((BASE_LABEL, base_python), (HEAD_LABEL, head_python))
-    for label, python in builds:
+    builds: tuple[tuple[str, Path, Path], ...] = (
+        (BASE_LABEL, base_python, base_project_dir),
+        (HEAD_LABEL, head_python, head_project_dir),
+    )
+    for label, python, project_dir in builds:
         _compile_once(label=label, python=python, project_dir=project_dir)
     results: dict[str, list[CompileRun]] = {BASE_LABEL: [], HEAD_LABEL: []}
     for _ in range(runs):
-        for label, python in builds:
+        for label, python, project_dir in builds:
             results[label].append(
                 _compile_once(label=label, python=python, project_dir=project_dir)
             )
@@ -88,11 +115,7 @@ def compare_builds(
 
 def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun:
     shutil.rmtree(project_dir / "target", ignore_errors=True)
-    environment: dict[str, str] = {
-        key: value
-        for key, value in os.environ.items()
-        if key not in EXCLUDED_ENVIRONMENT_KEYS and not key.startswith(EXCLUDED_ENVIRONMENT_PREFIX)
-    }
+    environment: dict[str, str] = _compile_environment()
     before: resource.struct_rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
     started: float = time.perf_counter()
     completed: subprocess.CompletedProcess[str] = subprocess.run(
@@ -116,7 +139,8 @@ def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun:
     after: resource.struct_rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
     if completed.returncode != 0:
         raise CompileComparisonError(
-            f"{label} compile failed with exit {completed.returncode}: {completed.stderr[-2000:]}"
+            f"{label} compile failed with exit {completed.returncode}: "
+            f"{completed.stderr[-ERROR_TAIL_CHARACTERS:]}"
         )
     payload: dict[str, object] = json.loads(completed.stdout)
     timings: object = payload.get("compile_timings", {})
@@ -132,6 +156,14 @@ def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun:
     )
 
 
+def _compile_environment() -> dict[str, str]:
+    return {
+        key: value
+        for key, value in os.environ.items()
+        if key not in EXCLUDED_ENVIRONMENT_KEYS and not key.startswith(EXCLUDED_ENVIRONMENT_PREFIX)
+    }
+
+
 def _median_phases(*, runs: list[CompileRun]) -> dict[str, float]:
     medians: dict[str, float] = {}
     for phase in REPORTED_PHASES:

scripts/compile_performance_ratio/_helpers/report.py

@@ -9,14 +9,23 @@
 from scripts.compile_performance_ratio.models import CompileComparison
 
 
-def comparison_markdown(*, comparison: CompileComparison, runs: int, max_ratio: float) -> str:
+def comparison_markdown(
+    *, comparison: CompileComparison, runs: int, max_ratio: float, per_side_projects: bool
+) -> str:
     """Summarize medians, ratios and per-phase timings as a Markdown table."""
 
+    generation: str = (
+        "Projects generated per side: base with the base checkout's generator, head with head's."
+        if per_side_projects
+        else "One project generated with head's generator and compiled by both builds."
+    )
     lines: list[str] = [
         f"### {comparison.kind} {comparison.models} models: head vs base (same runner)",
         "",
         f"Runner CPU: {cpu_model()}; {runs} alternating runs each; limit {max_ratio:.2f}x.",
         "",
+        generation,
+        "",
         "| Metric | Base | Head | Ratio |",
         "|---|---:|---:|---:|",
         f"| Wall (s) | {comparison.base_wall_seconds:.2f} | "

scripts/compile_performance_ratio/constants.py

@@ -10,6 +10,14 @@
 EXCLUDED_ENVIRONMENT_KEYS: frozenset[str] = frozenset({"VIRTUAL_ENV"})
 EXCLUDED_ENVIRONMENT_PREFIX: str = "DBT_"
 COMPILE_ENTRY: str = "import sys; from sqlbuild.cli.entry.main.entry import main; sys.exit(main())"
+BASE_GENERATOR_ENTRY: str = (
+    "import sys; from pathlib import Path; "
+    "from scripts.compile_performance_ratio._helpers.measure import write_benchmark_project; "
+    "write_benchmark_project("
+    "kind=sys.argv[1], project_dir=Path(sys.argv[2]), models=int(sys.argv[3]))"
+)
+PYTHONPATH_KEY: str = "PYTHONPATH"
+ERROR_TAIL_CHARACTERS: int = 2000
 FRESH_SOURCE_SHARE: float = 713 / 3000
 FRESH_SEED_SHARE: float = 141 / 3000
 FRESH_FUNCTION_SHARE: float = 71 / 3000