test(compiler): cover every render input in the compiler differential (#1001)

* test(compiler): generate every render input in the differential seeds

Add rendering feature blocks to the seeded generator: macros reading
context, constants, enums and the environment, nested and typed-reference
macro calls, macro-generated references with relaxed explicit-reference
rules, table functions, variable, environment and hook-context
interpolation, runtime placeholders, cursor intrinsics, macros in tests,
scenarios, audits, functions and source expressions, enum-typed columns,
model-private declarations, named hook arguments, and model, source and
seed audits. The harness provides the fixed environment variable the
generated projects read.

* test(compiler): require render coverage in the compiler differential

Classify CompileProjectInputs stage captures into render input kinds,
crediting a kind only from rendered evidence: a template absent from the
rendered SQL, a recorded usage, reference, grant, hook entry or attached
audit. Encoded empty sets and mappings prove nothing. Kinds credited from
a used macro's source text are reported as indirect.

Add --require-render-coverage and require it in the per-PR compiler
differential job. Share the capture readers with discovery coverage.

* test(compiler): add a render failure corpus

Add one failing project per reachable render, scope and attachment code
(P005, P007, P008, P009, S006, S007, S008, S009, K012, C214 and the C216
warning) and P001 cases for macro, interpolation, reference and hook
argument errors, each checked against its first error message.
render_error_codes derives the code set from the compiler; the end-to-end
test requires every code to be reached unless it is listed as
unreachable with a reason. Share the base-project case builder across
the corpus modules.

* test(compiler): prove compile-input stage captures are complete

Plan a generated project with every feature block and require its
CompileProjectInputs capture to hold every field canonically, name every
callable, and prove every required render kind. Perturb the native
compile inputs to show the differential localises a render mismatch to
the compile_project_inputs capture.

Author: kvlonge

.github/workflows/verify.yml

@@ -293,7 +293,8 @@ jobs:
       - name: Compare fixtures, examples, generated seeds and the failure corpus
         run: >-
           make compiler-differential
-          COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage"
+          COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage
+          --require-render-coverage"
 
   format_performance:
     name: Formatter performance guards

scripts/compiler_differential/_helpers/comparing/comparison.py

@@ -72,14 +72,28 @@ def compare_engine_runs(*, project: str, left: EngineRun, right: EngineRun) -> l
 def diagnostic_codes(*, outcome: CommandOutcome, severity: str | None) -> tuple[str, ...]:
     """Return a JSON compile report's diagnostic codes in order, optionally of one severity."""
 
+    return tuple(
+        str(diagnostic.get("code"))
+        for diagnostic in _diagnostics(outcome=outcome, severity=severity)
+    )
+
+
+def first_diagnostic_message(*, outcome: CommandOutcome, severity: str) -> str | None:
+    """Return the message of a JSON compile report's first diagnostic of one severity."""
+
+    diagnostics: list[dict[str, object]] = _diagnostics(outcome=outcome, severity=severity)
+    return str(diagnostics[0].get("message")) if diagnostics else None
+
+
+def _diagnostics(*, outcome: CommandOutcome, severity: str | None) -> list[dict[str, object]]:
     payload: dict[str, object] = as_json_object(_loads(outcome.stdout)) or {}
     diagnostics: object = payload.get("diagnostics")
-    found: list[str] = []
+    found: list[dict[str, object]] = []
     for raw_diagnostic in diagnostics if isinstance(diagnostics, list) else ():
         diagnostic: dict[str, object] | None = as_json_object(raw_diagnostic)
         if diagnostic is not None and (severity is None or diagnostic.get("severity") == severity):
-            found.append(str(diagnostic.get("code")))
-    return tuple(found)
+            found.append(diagnostic)
+    return found
 
 
 def _command_differences(

scripts/compiler_differential/_helpers/comparing/project_comparison.py

@@ -11,6 +11,7 @@
     diagnostic_codes,
 )
 from scripts.compiler_differential._helpers.coverage.discovery import project_discovery_kinds
+from scripts.compiler_differential._helpers.coverage.render import project_render_kinds
 from scripts.compiler_differential._helpers.running.execution import run_engine
 from scripts.compiler_differential.constants import ERROR_SEVERITY, WARNING_SEVERITY
 from scripts.compiler_differential.models import (
@@ -50,6 +51,7 @@ def compare_project(*, project: CorpusProject, options: DifferentialOptions) ->
         engine=right_engine,
         options=options,
     )
+    counted: bool = project.seed_coverage and options.stage_captures
     differences: list[Difference] = [
         *_timeout_differences(project=project, runs=(left, right)),
         *_corpus_differences(project=project, run=left),
@@ -61,9 +63,10 @@ def compare_project(*, project: CorpusProject, options: DifferentialOptions) ->
         seconds=time.monotonic() - started,
         discovered_kinds=(
             project_discovery_kinds(captures=left.captures, source_dir=source_dir)
-            if project.discovery_coverage and options.stage_captures
+            if counted
             else frozenset()
         ),
+        rendered_kinds=project_render_kinds(left.captures) if counted else frozenset(),
     )
 
 

scripts/compiler_differential/_helpers/corpus/case_builder.py

@@ -0,0 +1,49 @@
+"""Build failure cases as small edits of the shared base project."""
+
+from scripts.compiler_differential.constants import (
+    FAILURE_BASE_CONFIG,
+    FAILURE_BASE_FILES,
+    FAILURE_CONFIG_PATH,
+    FAILURE_MART_PATH,
+    FAILURE_STAGING_PATH,
+)
+from scripts.compiler_differential.models import FailureCase
+
+_MART_HEADER: str = 'MODEL (\n  description "Order totals per customer",\n);\n\n'
+
+
+def failure_case(
+    *,
+    name: str,
+    expected_code: str,
+    files: dict[str, str],
+    expected_warning_code: str | None = None,
+    expected_message: str | None = None,
+) -> FailureCase:
+    """Return the base project with `files` added or replaced."""
+
+    return FailureCase(
+        files={**FAILURE_BASE_FILES, **files},
+        name=name,
+        expected_code=expected_code,
+        expected_warning_code=expected_warning_code,
+        expected_message=expected_message,
+    )
+
+
+def staging_files(sql: str) -> dict[str, str]:
+    """Replace the staging model."""
+
+    return {FAILURE_STAGING_PATH: sql}
+
+
+def mart_body_files(body: str) -> dict[str, str]:
+    """Replace the mart model's query, keeping its header."""
+
+    return {FAILURE_MART_PATH: _MART_HEADER + body}
+
+
+def config_files(extra: str) -> dict[str, str]:
+    """Append `extra` to the base project configuration."""
+
+    return {FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + extra}

scripts/compiler_differential/_helpers/corpus/corpus.py

@@ -143,5 +143,5 @@ def _seed_project(seed: int) -> CorpusProject:
             succeeding_commands=generated.succeeding_commands,
         ),
         writer=generated.write,
-        discovery_coverage=True,
+        seed_coverage=True,
     )