test(compiler): cover every render input in the compiler differential

.github/workflows/verify.yml

@@ -293,7 +293,8 @@ jobs:
       - name: Compare fixtures, examples, generated seeds and the failure corpus
         run: >-
           make compiler-differential
-          COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage"
+          COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage
+          --require-render-coverage"
 
   format_performance:
     name: Formatter performance guards

scripts/compiler_differential/_helpers/comparing/comparison.py

@@ -72,14 +72,28 @@ def compare_engine_runs(*, project: str, left: EngineRun, right: EngineRun) -> l
 def diagnostic_codes(*, outcome: CommandOutcome, severity: str | None) -> tuple[str, ...]:
     """Return a JSON compile report's diagnostic codes in order, optionally of one severity."""
 
+    return tuple(
+        str(diagnostic.get("code"))
+        for diagnostic in _diagnostics(outcome=outcome, severity=severity)
+    )
+
+
+def first_diagnostic_message(*, outcome: CommandOutcome, severity: str) -> str | None:
+    """Return the message of a JSON compile report's first diagnostic of one severity."""
+
+    diagnostics: list[dict[str, object]] = _diagnostics(outcome=outcome, severity=severity)
+    return str(diagnostics[0].get("message")) if diagnostics else None
+
+
+def _diagnostics(*, outcome: CommandOutcome, severity: str | None) -> list[dict[str, object]]:
     payload: dict[str, object] = as_json_object(_loads(outcome.stdout)) or {}
     diagnostics: object = payload.get("diagnostics")
-    found: list[str] = []
+    found: list[dict[str, object]] = []
     for raw_diagnostic in diagnostics if isinstance(diagnostics, list) else ():
         diagnostic: dict[str, object] | None = as_json_object(raw_diagnostic)
         if diagnostic is not None and (severity is None or diagnostic.get("severity") == severity):
-            found.append(str(diagnostic.get("code")))
-    return tuple(found)
+            found.append(diagnostic)
+    return found
 
 
 def _command_differences(

scripts/compiler_differential/_helpers/comparing/project_comparison.py

@@ -11,6 +11,7 @@
     diagnostic_codes,
 )
 from scripts.compiler_differential._helpers.coverage.discovery import project_discovery_kinds
+from scripts.compiler_differential._helpers.coverage.render import project_render_kinds
 from scripts.compiler_differential._helpers.running.execution import run_engine
 from scripts.compiler_differential.constants import ERROR_SEVERITY, WARNING_SEVERITY
 from scripts.compiler_differential.models import (
@@ -50,6 +51,7 @@ def compare_project(*, project: CorpusProject, options: DifferentialOptions) ->
         engine=right_engine,
         options=options,
     )
+    counted: bool = project.seed_coverage and options.stage_captures
     differences: list[Difference] = [
         *_timeout_differences(project=project, runs=(left, right)),
         *_corpus_differences(project=project, run=left),
@@ -61,9 +63,10 @@ def compare_project(*, project: CorpusProject, options: DifferentialOptions) ->
         seconds=time.monotonic() - started,
         discovered_kinds=(
             project_discovery_kinds(captures=left.captures, source_dir=source_dir)
-            if project.discovery_coverage and options.stage_captures
+            if counted
             else frozenset()
         ),
+        rendered_kinds=project_render_kinds(left.captures) if counted else frozenset(),
     )
 
 

scripts/compiler_differential/_helpers/corpus/case_builder.py

@@ -0,0 +1,49 @@
+"""Build failure cases as small edits of the shared base project."""
+
+from scripts.compiler_differential.constants import (
+    FAILURE_BASE_CONFIG,
+    FAILURE_BASE_FILES,
+    FAILURE_CONFIG_PATH,
+    FAILURE_MART_PATH,
+    FAILURE_STAGING_PATH,
+)
+from scripts.compiler_differential.models import FailureCase
+
+_MART_HEADER: str = 'MODEL (\n  description "Order totals per customer",\n);\n\n'
+
+
+def failure_case(
+    *,
+    name: str,
+    expected_code: str,
+    files: dict[str, str],
+    expected_warning_code: str | None = None,
+    expected_message: str | None = None,
+) -> FailureCase:
+    """Return the base project with `files` added or replaced."""
+
+    return FailureCase(
+        files={**FAILURE_BASE_FILES, **files},
+        name=name,
+        expected_code=expected_code,
+        expected_warning_code=expected_warning_code,
+        expected_message=expected_message,
+    )
+
+
+def staging_files(sql: str) -> dict[str, str]:
+    """Replace the staging model."""
+
+    return {FAILURE_STAGING_PATH: sql}
+
+
+def mart_body_files(body: str) -> dict[str, str]:
+    """Replace the mart model's query, keeping its header."""
+
+    return {FAILURE_MART_PATH: _MART_HEADER + body}
+
+
+def config_files(extra: str) -> dict[str, str]:
+    """Append `extra` to the base project configuration."""
+
+    return {FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + extra}

scripts/compiler_differential/_helpers/corpus/corpus.py

@@ -143,5 +143,5 @@ def _seed_project(seed: int) -> CorpusProject:
             succeeding_commands=generated.succeeding_commands,
         ),
         writer=generated.write,
-        discovery_coverage=True,
+        seed_coverage=True,
     )