test(compiler): cover every discovery input in the compiler differential

.github/workflows/verify.yml

@@ -291,7 +291,9 @@ jobs:
       - name: Install dependencies
         run: uv sync --locked --all-groups --all-extras
       - name: Compare fixtures, examples, generated seeds and the failure corpus
-        run: make compiler-differential COMPILER_DIFFERENTIAL_ARGS=--stage-captures
+        run: >-
+          make compiler-differential
+          COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage"
 
   format_performance:
     name: Formatter performance guards

scripts/compiler_differential/_helpers/comparing/__init__.py

@@ -0,0 +1 @@
+"""Compare what two engines produced and locate the first differences."""

scripts/compiler_differential/_helpers/comparison.py → scripts/compiler_differential/_helpers/comparing/comparison.py

@@ -5,20 +5,19 @@
 import json
 from collections.abc import Callable
 
-from scripts.compiler_differential._helpers.compare import (
+from scripts.compiler_differential._helpers.comparing.compare import (
     as_json_object,
     first_document_difference,
     first_json_difference,
     first_text_difference,
 )
-from scripts.compiler_differential._helpers.normalize import (
+from scripts.compiler_differential._helpers.comparing.normalize import (
     normalize_artifact_text,
     normalize_stderr,
     strip_manifest_metadata,
     strip_report_fields,
 )
 from scripts.compiler_differential.constants import (
-    ERROR_SEVERITY,
     MISSING_VALUE,
     MODEL_RESOURCE_TYPE,
     PLAN_LABEL,
@@ -70,17 +69,15 @@ def compare_engine_runs(*, project: str, left: EngineRun, right: EngineRun) -> l
     ]
 
 
-def diagnostic_codes(*, outcome: CommandOutcome, errors_only: bool) -> tuple[str, ...]:
-    """Return the diagnostic codes of a JSON compile report, in report order."""
+def diagnostic_codes(*, outcome: CommandOutcome, severity: str | None) -> tuple[str, ...]:
+    """Return a JSON compile report's diagnostic codes in order, optionally of one severity."""
 
     payload: dict[str, object] = as_json_object(_loads(outcome.stdout)) or {}
     diagnostics: object = payload.get("diagnostics")
     found: list[str] = []
     for raw_diagnostic in diagnostics if isinstance(diagnostics, list) else ():
         diagnostic: dict[str, object] | None = as_json_object(raw_diagnostic)
-        if diagnostic is not None and (
-            not errors_only or diagnostic.get("severity") == ERROR_SEVERITY
-        ):
+        if diagnostic is not None and (severity is None or diagnostic.get("severity") == severity):
             found.append(str(diagnostic.get("code")))
     return tuple(found)
 

scripts/compiler_differential/_helpers/normalize.py → scripts/compiler_differential/_helpers/comparing/normalize.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from scripts.compiler_differential._helpers.compare import as_json_object
+from scripts.compiler_differential._helpers.comparing.compare import as_json_object
 from scripts.compiler_differential.constants import (
     ELAPSED_TIME_MASK,
     ELAPSED_TIME_PATTERN,

scripts/compiler_differential/_helpers/project_comparison.py → scripts/compiler_differential/_helpers/comparing/project_comparison.py

@@ -6,8 +6,13 @@
 import time
 from pathlib import Path
 
-from scripts.compiler_differential._helpers.comparison import compare_engine_runs, diagnostic_codes
-from scripts.compiler_differential._helpers.execution import run_engine
+from scripts.compiler_differential._helpers.comparing.comparison import (
+    compare_engine_runs,
+    diagnostic_codes,
+)
+from scripts.compiler_differential._helpers.coverage.discovery import project_discovery_kinds
+from scripts.compiler_differential._helpers.running.execution import run_engine
+from scripts.compiler_differential.constants import ERROR_SEVERITY, WARNING_SEVERITY
 from scripts.compiler_differential.models import (
     CommandOutcome,
     CorpusProject,
@@ -54,6 +59,11 @@ def compare_project(*, project: CorpusProject, options: DifferentialOptions) ->
         project=project.name,
         differences=tuple(differences),
         seconds=time.monotonic() - started,
+        discovered_kinds=(
+            project_discovery_kinds(captures=left.captures, source_dir=source_dir)
+            if project.discovery_coverage and options.stage_captures
+            else frozenset()
+        ),
     )
 
 
@@ -68,23 +78,21 @@ def _materialized_source(*, project: CorpusProject, case_dir: Path) -> Path:
 
 def _corpus_differences(*, project: CorpusProject, run: EngineRun) -> list[Difference]:
     first: CommandOutcome = run.outcomes[0]
-    errors: tuple[str, ...] = diagnostic_codes(outcome=first, errors_only=True)
-    expected_code: str | None = project.expected.error_code
-    if expected_code is not None:
-        codes: tuple[str, ...] = diagnostic_codes(outcome=first, errors_only=False)
-        met: bool = first.exit_code != 0 and bool(errors) and expected_code in codes
-        return (
-            []
-            if met
-            else [
+    errors: tuple[str, ...] = diagnostic_codes(outcome=first, severity=ERROR_SEVERITY)
+    if project.expected.error_code is not None:
+        return [
+            *_failure_differences(project=project, outcome=first, errors=errors),
+            *(
                 _expectation_difference(
                     project=project,
-                    location="expected failure",
-                    expected=f"non-zero exit reporting {expected_code}",
-                    actual=f"exit {first.exit_code}: {', '.join(codes) or '<no codes>'}",
+                    location=f"`{outcome.label}` expected success",
+                    expected="exit 0",
+                    actual=f"exit {outcome.exit_code}",
                 )
-            ]
-        )
+                for outcome in run.outcomes
+                if outcome.label in project.expected.succeeding_commands and outcome.exit_code != 0
+            ),
+        ]
     problems: list[str] = [
         f"`{outcome.label}` exit {outcome.exit_code}"
         for outcome in run.outcomes
@@ -104,6 +112,34 @@ def _corpus_differences(*, project: CorpusProject, run: EngineRun) -> list[Diffe
     ]
 
 
+def _failure_differences(
+    *, project: CorpusProject, outcome: CommandOutcome, errors: tuple[str, ...]
+) -> list[Difference]:
+    expected_code: str | None = project.expected.error_code
+    differences: list[Difference] = []
+    if outcome.exit_code == 0 or not errors or errors[0] != expected_code:
+        differences.append(
+            _expectation_difference(
+                project=project,
+                location="expected failure",
+                expected=f"non-zero exit whose first error is {expected_code}",
+                actual=f"exit {outcome.exit_code}: errors {', '.join(errors) or '<none>'}",
+            )
+        )
+    warning_code: str | None = project.expected.warning_code
+    warnings: tuple[str, ...] = diagnostic_codes(outcome=outcome, severity=WARNING_SEVERITY)
+    if warning_code is not None and warning_code not in warnings:
+        differences.append(
+            _expectation_difference(
+                project=project,
+                location="expected warning",
+                expected=f"warning {warning_code}",
+                actual=f"warnings {', '.join(warnings) or '<none>'}",
+            )
+        )
+    return differences
+
+
 def _timeout_differences(
     *, project: CorpusProject, runs: tuple[EngineRun, ...]
 ) -> list[Difference]:

scripts/compiler_differential/_helpers/corpus/__init__.py

@@ -0,0 +1 @@
+"""Assemble the corpus projects, including the failure corpus."""

scripts/compiler_differential/_helpers/corpus.py → scripts/compiler_differential/_helpers/corpus/corpus.py

@@ -5,7 +5,7 @@
 import tomllib
 from pathlib import Path
 
-from scripts.compiler_differential._helpers.failure_cases import all_failure_cases
+from scripts.compiler_differential._helpers.corpus.failure_cases import all_failure_cases
 from scripts.compiler_differential.classes.dense_project import DenseProject
 from scripts.compiler_differential.classes.project_builder import ProjectBuilder
 from scripts.compiler_differential.constants import (
@@ -80,7 +80,9 @@ def build_corpus(
             CorpusProject(
                 name=f"failure/{case.name}",
                 commands=(COLD_COMPILE,),
-                expected=ExpectedOutcome(error_code=case.expected_code),
+                expected=ExpectedOutcome(
+                    error_code=case.expected_code, warning_code=case.expected_warning_code
+                ),
                 writer=case.write,
             )
             for case in all_failure_cases()
@@ -136,6 +138,10 @@ def _seed_project(seed: int) -> CorpusProject:
     return CorpusProject(
         name=f"seed/{seed}",
         commands=(COLD_COMPILE, PLAN),
-        expected=ExpectedOutcome(error_code=generated.expected_error_code),
+        expected=ExpectedOutcome(
+            error_code=generated.expected_error_code,
+            succeeding_commands=generated.succeeding_commands,
+        ),
         writer=generated.write,
+        discovery_coverage=True,
     )