ci: keep compiler differential failure evidence

.github/workflows/verify.yml

@@ -308,7 +308,8 @@ jobs:
           make compiler-differential
           COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage
           --require-render-coverage --require-analysis-coverage
-          --wheel-site-report ${{ runner.temp }}/compiler-differential-wheel-sites.json"
+          --wheel-site-report ${{ runner.temp }}/compiler-differential-wheel-sites.json
+          --evidence-dir ${{ runner.temp }}/compiler-differential-evidence/preview"
       - name: Upload the Polyglot wheel-site and deferral report
         if: always()
         uses: actions/upload-artifact@v4
@@ -321,7 +322,16 @@ jobs:
         run: >-
           make compiler-differential-shipped
           COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage
-          --require-render-coverage"
+          --require-render-coverage
+          --evidence-dir ${{ runner.temp }}/compiler-differential-evidence/shipped"
+      - name: Upload differing projects' targets, stderr and compiled-test stats
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: compiler-differential-evidence-${{ github.run_id }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/compiler-differential-evidence
+          if-no-files-found: warn
+          retention-days: 30
 
   compiler_differential_python_versions:
     name: Native compiler on Python ${{ matrix.python }}

scripts/compiler_differential/_helpers/comparing/project_comparison.py

@@ -13,6 +13,7 @@
 from scripts.compiler_differential._helpers.coverage.analysis import project_analysis_kinds
 from scripts.compiler_differential._helpers.coverage.discovery import project_discovery_kinds
 from scripts.compiler_differential._helpers.coverage.render import project_render_kinds
+from scripts.compiler_differential._helpers.running.evidence import keep_failure_evidence
 from scripts.compiler_differential._helpers.running.execution import run_engine
 from scripts.compiler_differential.constants import (
     ERROR_SEVERITY,
@@ -65,6 +66,13 @@ def compare_project(*, project: CorpusProject, options: DifferentialOptions) ->
         *_corpus_differences(project=project, run=left),
         *compare_engine_runs(project=project.name, left=left, right=right),
     ]
+    if differences and options.evidence_dir is not None:
+        _ = keep_failure_evidence(
+            case_dir=case_dir,
+            project_subdirectory=project.project_subdirectory,
+            runs=(left, right),
+            evidence_dir=options.evidence_dir,
+        )
     return ProjectComparison(
         project=project.name,
         differences=tuple(differences),

scripts/compiler_differential/_helpers/running/evidence.py

@@ -0,0 +1,56 @@
+"""Keep a differing project's engine runs where CI can upload them."""
+
+from __future__ import annotations
+
+import os
+import shutil
+from pathlib import Path
+
+from scripts.compiler_differential.models import EngineRun
+
+_TARGET_DIRECTORY: str = "target"
+_COMPILED_TESTS: tuple[str, ...] = ("target", "compiled", "tests")
+_STAT_FILE: str = "compiled-tests-stat.tsv"
+
+
+def keep_failure_evidence(
+    *,
+    case_dir: Path,
+    project_subdirectory: str | None,
+    runs: tuple[EngineRun, ...],
+    evidence_dir: Path,
+) -> Path:
+    """Copy each engine's `target/`, every command's stderr and a compiled-test stat listing."""
+
+    destination: Path = evidence_dir / case_dir.name
+    shutil.rmtree(destination, ignore_errors=True)
+    for run in runs:
+        project_dir: Path = case_dir / run.engine / (project_subdirectory or "")
+        engine_dir: Path = destination / run.engine
+        engine_dir.mkdir(parents=True)
+        target: Path = project_dir / _TARGET_DIRECTORY
+        if target.is_dir():
+            _ = shutil.copytree(target, engine_dir / _TARGET_DIRECTORY, symlinks=True)
+        for index, outcome in enumerate(run.outcomes):
+            _ = (engine_dir / f"{index}-{outcome.label}.stderr").write_text(
+                outcome.stderr, encoding="utf-8", errors="surrogateescape"
+            )
+        _ = (engine_dir / _STAT_FILE).write_text(
+            _stat_listing(project_dir.joinpath(*_COMPILED_TESTS)), encoding="utf-8"
+        )
+    return destination
+
+
+def _stat_listing(root: Path) -> str:
+    """`relative path, size, mtime_ns, ctime_ns` for every file below `root`, sorted."""
+
+    rows: list[str] = ["path\tsize\tmtime_ns\tctime_ns"]
+    for directory, _, filenames in sorted(os.walk(root)):
+        for filename in sorted(filenames):
+            path: Path = Path(directory, filename)
+            status: os.stat_result = path.stat()
+            rows.append(
+                f"{path.relative_to(root).as_posix()}\t{status.st_size}\t"
+                f"{status.st_mtime_ns}\t{status.st_ctime_ns}"
+            )
+    return "\n".join(rows) + "\n"

scripts/compiler_differential/_helpers/running/run.py

@@ -32,6 +32,7 @@ def differential_options(*, args: argparse.Namespace, work_dir: Path) -> Differe
         require_render_coverage=args.require_render_coverage,
         require_analysis_coverage=args.require_analysis_coverage,
         analysis_records=args.wheel_site_report is not None,
+        evidence_dir=args.evidence_dir,
     )
 
 

scripts/compiler_differential/main/differential.py

@@ -187,6 +187,13 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace:
         help="extra environment for one engine's processes",
     )
     parser.add_argument("--work-dir", type=Path, default=None, help="keep run directories here")
+    parser.add_argument(
+        "--evidence-dir",
+        type=Path,
+        default=None,
+        metavar="PATH",
+        help="copy each differing project's target/, stderr and compiled-test stats to PATH",
+    )
     parser.add_argument("--python", type=Path, default=Path(sys.executable))
     args: argparse.Namespace = parser.parse_args(argv)
     for flag, required in (

scripts/compiler_differential/models.py

@@ -118,6 +118,7 @@ class DifferentialOptions:
     require_render_coverage: bool = False
     require_analysis_coverage: bool = False
     analysis_records: bool = False
+    evidence_dir: Path | None = None
 
 
 @dataclass(frozen=True)

tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py

@@ -75,3 +75,13 @@ class AnalysisRecordsTestCase:
     expected_wheel_sites: dict[tuple[str, str], int]
     expected_deferrals: dict[tuple[str, str], int]
     expected_lines: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class FailureEvidenceTestCase:
+    """Per-engine warm-compile stderr and the evidence files a comparison keeps."""
+
+    description: str
+    stderr_by_engine: dict[str, str]
+    expected_files: tuple[str, ...]
+    expected_evidence_engines: tuple[str, ...]

tests/unit/scripts/compiler_differential/_helpers/running/helpers.py

@@ -0,0 +1,72 @@
+"""Fake engine runs for the differential harness running tests."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from pathlib import Path
+
+from scripts.compiler_differential.models import (
+    CommandOutcome,
+    CorpusProject,
+    DifferentialOptions,
+    EngineRun,
+)
+
+ORDERS_TEST_SQL: str = "SELECT order_id FROM orders\n"
+
+
+def fake_run_engine(*, stderr_by_engine: dict[str, str]) -> Callable[..., EngineRun]:
+    """Return a `run_engine` that leaves one compiled test and prints the engine's stderr."""
+
+    def run_engine(
+        *,
+        project: CorpusProject,
+        source_dir: Path,
+        case_dir: Path,
+        engine: str,
+        side: str,
+        options: DifferentialOptions,
+    ) -> EngineRun:
+        del project, source_dir, side, options
+        tests_dir: Path = case_dir / engine / "target" / "compiled" / "tests" / "orders"
+        tests_dir.mkdir(parents=True)
+        _ = (tests_dir / "test_orders.sql").write_text(ORDERS_TEST_SQL, encoding="utf-8")
+        _ = (case_dir / engine / "target" / "sql-test-artifacts.json").write_text(
+            "{}", encoding="utf-8"
+        )
+        return EngineRun(
+            engine=engine,
+            outcomes=(
+                CommandOutcome(
+                    label="compile-warm",
+                    exit_code=0,
+                    stdout="{}",
+                    stderr=stderr_by_engine[engine],
+                ),
+            ),
+            compiled={},
+            manifest=None,
+            dag=None,
+        )
+
+    return run_engine
+
+
+def first_stat_paths(*, evidence_dir: Path, engines: tuple[str, ...]) -> list[str]:
+    """The first compiled-test path each engine's stat listing records."""
+
+    return [
+        (evidence_dir / "example__orders" / engine / "compiled-tests-stat.tsv")
+        .read_text(encoding="utf-8")
+        .splitlines()[1]
+        .split("\t")[0]
+        for engine in engines
+    ]
+
+
+def kept_files(root: Path) -> tuple[str, ...]:
+    """Every file below `root` as sorted relative POSIX paths."""
+
+    return tuple(
+        sorted(path.relative_to(root).as_posix() for path in filter(Path.is_file, root.rglob("*")))
+    )

tests/unit/scripts/compiler_differential/_helpers/running/test_evidence.py

@@ -0,0 +1,94 @@
+"""A differing project keeps both engines' target, stderr and compiled-test stats."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from scripts.compiler_differential._helpers.comparing import project_comparison
+from scripts.compiler_differential.models import (
+    CorpusProject,
+    DifferentialCommand,
+    DifferentialOptions,
+    ExpectedOutcome,
+    ProjectComparison,
+)
+from tests.unit.scripts.compiler_differential._helpers.running._test_types import (
+    FailureEvidenceTestCase,
+)
+from tests.unit.scripts.compiler_differential._helpers.running.helpers import (
+    fake_run_engine,
+    first_stat_paths,
+    kept_files,
+)
+
+
+@pytest.mark.parametrize(
+    "test_case",
+    [
+        FailureEvidenceTestCase(
+            description="differing warm stderr keeps both engines' evidence",
+            stderr_by_engine={
+                "python": "Semantic checks were partial for 1 models:\n",
+                "native-preview": "SQL test planning  START\n",
+            },
+            expected_files=(
+                "example__orders/native-preview/0-compile-warm.stderr",
+                "example__orders/native-preview/compiled-tests-stat.tsv",
+                "example__orders/native-preview/target/compiled/tests/orders/test_orders.sql",
+                "example__orders/native-preview/target/sql-test-artifacts.json",
+                "example__orders/python/0-compile-warm.stderr",
+                "example__orders/python/compiled-tests-stat.tsv",
+                "example__orders/python/target/compiled/tests/orders/test_orders.sql",
+                "example__orders/python/target/sql-test-artifacts.json",
+            ),
+            expected_evidence_engines=("native-preview", "python"),
+        ),
+        FailureEvidenceTestCase(
+            description="identical runs keep nothing",
+            stderr_by_engine={"python": "done\n", "native-preview": "done\n"},
+            expected_files=(),
+            expected_evidence_engines=(),
+        ),
+    ],
+    ids=lambda case: case.description,
+)
+def test_given_engine_runs_when_comparing_then_only_differing_projects_keep_evidence(
+    test_case: FailureEvidenceTestCase,
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(
+        project_comparison,
+        "run_engine",
+        fake_run_engine(stderr_by_engine=test_case.stderr_by_engine),
+    )
+    evidence_dir: Path = tmp_path / "evidence"
+
+    comparison: ProjectComparison = project_comparison.compare_project(
+        project=CorpusProject(
+            name="example/orders",
+            commands=(DifferentialCommand(label="compile-warm", arguments=("compile",)),),
+            expected=ExpectedOutcome(),
+            source_dir=tmp_path,
+        ),
+        options=DifferentialOptions(
+            engines=("python", "native-preview"),
+            work_dir=tmp_path / "work",
+            jobs=1,
+            stage_captures=False,
+            python=Path("python"),
+            engine_environment={},
+            evidence_dir=evidence_dir,
+        ),
+    )
+    stats: list[str] = first_stat_paths(
+        evidence_dir=evidence_dir, engines=test_case.expected_evidence_engines
+    )
+
+    assert (bool(comparison.differences), kept_files(evidence_dir), stats) == (
+        bool(test_case.expected_files),
+        test_case.expected_files,
+        ["orders/test_orders.sql"] * len(test_case.expected_evidence_engines),
+    )