From 55f7d0f6ff1a9dbb92aecde2a180c8f91a8ff4de Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:27:02 +0100 Subject: [PATCH] ci: keep compiler differential failure evidence --- .github/workflows/verify.yml | 14 ++- .../_helpers/comparing/project_comparison.py | 8 ++ .../_helpers/running/evidence.py | 56 +++++++++++ .../_helpers/running/run.py | 1 + .../main/differential.py | 7 ++ scripts/compiler_differential/models.py | 1 + .../_helpers/running/_test_types.py | 10 ++ .../_helpers/running/helpers.py | 72 ++++++++++++++ .../_helpers/running/test_evidence.py | 94 +++++++++++++++++++ 9 files changed, 261 insertions(+), 2 deletions(-) create mode 100644 scripts/compiler_differential/_helpers/running/evidence.py create mode 100644 tests/unit/scripts/compiler_differential/_helpers/running/helpers.py create mode 100644 tests/unit/scripts/compiler_differential/_helpers/running/test_evidence.py diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index ccc2a8763d..477b06fe62 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -308,7 +308,8 @@ jobs: make compiler-differential COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage --require-render-coverage --require-analysis-coverage - --wheel-site-report ${{ runner.temp }}/compiler-differential-wheel-sites.json" + --wheel-site-report ${{ runner.temp }}/compiler-differential-wheel-sites.json + --evidence-dir ${{ runner.temp }}/compiler-differential-evidence/preview" - name: Upload the Polyglot wheel-site and deferral report if: always() uses: actions/upload-artifact@v4 @@ -321,7 +322,16 @@ jobs: run: >- make compiler-differential-shipped COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage - --require-render-coverage" + --require-render-coverage + --evidence-dir ${{ runner.temp }}/compiler-differential-evidence/shipped" + - name: Upload differing projects' targets, stderr and compiled-test stats + if: failure() + uses: actions/upload-artifact@v4 + with: + name: compiler-differential-evidence-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/compiler-differential-evidence + if-no-files-found: warn + retention-days: 30 compiler_differential_python_versions: name: Native compiler on Python ${{ matrix.python }} diff --git a/scripts/compiler_differential/_helpers/comparing/project_comparison.py b/scripts/compiler_differential/_helpers/comparing/project_comparison.py index ac14b3d1fd..f6c4227b1e 100644 --- a/scripts/compiler_differential/_helpers/comparing/project_comparison.py +++ b/scripts/compiler_differential/_helpers/comparing/project_comparison.py @@ -13,6 +13,7 @@ from scripts.compiler_differential._helpers.coverage.analysis import project_analysis_kinds from scripts.compiler_differential._helpers.coverage.discovery import project_discovery_kinds from scripts.compiler_differential._helpers.coverage.render import project_render_kinds +from scripts.compiler_differential._helpers.running.evidence import keep_failure_evidence from scripts.compiler_differential._helpers.running.execution import run_engine from scripts.compiler_differential.constants import ( ERROR_SEVERITY, @@ -65,6 +66,13 @@ def compare_project(*, project: CorpusProject, options: DifferentialOptions) -> *_corpus_differences(project=project, run=left), *compare_engine_runs(project=project.name, left=left, right=right), ] + if differences and options.evidence_dir is not None: + _ = keep_failure_evidence( + case_dir=case_dir, + project_subdirectory=project.project_subdirectory, + runs=(left, right), + evidence_dir=options.evidence_dir, + ) return ProjectComparison( project=project.name, differences=tuple(differences), diff --git a/scripts/compiler_differential/_helpers/running/evidence.py b/scripts/compiler_differential/_helpers/running/evidence.py new file mode 100644 index 0000000000..294ff1d15d --- /dev/null +++ b/scripts/compiler_differential/_helpers/running/evidence.py @@ -0,0 +1,56 @@ +"""Keep a differing project's engine runs where CI can upload them.""" + +from __future__ import annotations + +import os +import shutil +from pathlib import Path + +from scripts.compiler_differential.models import EngineRun + +_TARGET_DIRECTORY: str = "target" +_COMPILED_TESTS: tuple[str, ...] = ("target", "compiled", "tests") +_STAT_FILE: str = "compiled-tests-stat.tsv" + + +def keep_failure_evidence( + *, + case_dir: Path, + project_subdirectory: str | None, + runs: tuple[EngineRun, ...], + evidence_dir: Path, +) -> Path: + """Copy each engine's `target/`, every command's stderr and a compiled-test stat listing.""" + + destination: Path = evidence_dir / case_dir.name + shutil.rmtree(destination, ignore_errors=True) + for run in runs: + project_dir: Path = case_dir / run.engine / (project_subdirectory or "") + engine_dir: Path = destination / run.engine + engine_dir.mkdir(parents=True) + target: Path = project_dir / _TARGET_DIRECTORY + if target.is_dir(): + _ = shutil.copytree(target, engine_dir / _TARGET_DIRECTORY, symlinks=True) + for index, outcome in enumerate(run.outcomes): + _ = (engine_dir / f"{index}-{outcome.label}.stderr").write_text( + outcome.stderr, encoding="utf-8", errors="surrogateescape" + ) + _ = (engine_dir / _STAT_FILE).write_text( + _stat_listing(project_dir.joinpath(*_COMPILED_TESTS)), encoding="utf-8" + ) + return destination + + +def _stat_listing(root: Path) -> str: + """`relative path, size, mtime_ns, ctime_ns` for every file below `root`, sorted.""" + + rows: list[str] = ["path\tsize\tmtime_ns\tctime_ns"] + for directory, _, filenames in sorted(os.walk(root)): + for filename in sorted(filenames): + path: Path = Path(directory, filename) + status: os.stat_result = path.stat() + rows.append( + f"{path.relative_to(root).as_posix()}\t{status.st_size}\t" + f"{status.st_mtime_ns}\t{status.st_ctime_ns}" + ) + return "\n".join(rows) + "\n" diff --git a/scripts/compiler_differential/_helpers/running/run.py b/scripts/compiler_differential/_helpers/running/run.py index d6dbbb668f..b1661e72b0 100644 --- a/scripts/compiler_differential/_helpers/running/run.py +++ b/scripts/compiler_differential/_helpers/running/run.py @@ -32,6 +32,7 @@ def differential_options(*, args: argparse.Namespace, work_dir: Path) -> Differe require_render_coverage=args.require_render_coverage, require_analysis_coverage=args.require_analysis_coverage, analysis_records=args.wheel_site_report is not None, + evidence_dir=args.evidence_dir, ) diff --git a/scripts/compiler_differential/main/differential.py b/scripts/compiler_differential/main/differential.py index 446b25b09e..c755a12f6f 100644 --- a/scripts/compiler_differential/main/differential.py +++ b/scripts/compiler_differential/main/differential.py @@ -187,6 +187,13 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: help="extra environment for one engine's processes", ) parser.add_argument("--work-dir", type=Path, default=None, help="keep run directories here") + parser.add_argument( + "--evidence-dir", + type=Path, + default=None, + metavar="PATH", + help="copy each differing project's target/, stderr and compiled-test stats to PATH", + ) parser.add_argument("--python", type=Path, default=Path(sys.executable)) args: argparse.Namespace = parser.parse_args(argv) for flag, required in ( diff --git a/scripts/compiler_differential/models.py b/scripts/compiler_differential/models.py index 0a9cd20d14..8de8914cb7 100644 --- a/scripts/compiler_differential/models.py +++ b/scripts/compiler_differential/models.py @@ -118,6 +118,7 @@ class DifferentialOptions: require_render_coverage: bool = False require_analysis_coverage: bool = False analysis_records: bool = False + evidence_dir: Path | None = None @dataclass(frozen=True) diff --git a/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py index 6233c40c85..f9054ef85e 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py +++ b/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py @@ -75,3 +75,13 @@ class AnalysisRecordsTestCase: expected_wheel_sites: dict[tuple[str, str], int] expected_deferrals: dict[tuple[str, str], int] expected_lines: tuple[str, ...] + + +@dataclass(frozen=True) +class FailureEvidenceTestCase: + """Per-engine warm-compile stderr and the evidence files a comparison keeps.""" + + description: str + stderr_by_engine: dict[str, str] + expected_files: tuple[str, ...] + expected_evidence_engines: tuple[str, ...] diff --git a/tests/unit/scripts/compiler_differential/_helpers/running/helpers.py b/tests/unit/scripts/compiler_differential/_helpers/running/helpers.py new file mode 100644 index 0000000000..a5c39c5397 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/running/helpers.py @@ -0,0 +1,72 @@ +"""Fake engine runs for the differential harness running tests.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +from scripts.compiler_differential.models import ( + CommandOutcome, + CorpusProject, + DifferentialOptions, + EngineRun, +) + +ORDERS_TEST_SQL: str = "SELECT order_id FROM orders\n" + + +def fake_run_engine(*, stderr_by_engine: dict[str, str]) -> Callable[..., EngineRun]: + """Return a `run_engine` that leaves one compiled test and prints the engine's stderr.""" + + def run_engine( + *, + project: CorpusProject, + source_dir: Path, + case_dir: Path, + engine: str, + side: str, + options: DifferentialOptions, + ) -> EngineRun: + del project, source_dir, side, options + tests_dir: Path = case_dir / engine / "target" / "compiled" / "tests" / "orders" + tests_dir.mkdir(parents=True) + _ = (tests_dir / "test_orders.sql").write_text(ORDERS_TEST_SQL, encoding="utf-8") + _ = (case_dir / engine / "target" / "sql-test-artifacts.json").write_text( + "{}", encoding="utf-8" + ) + return EngineRun( + engine=engine, + outcomes=( + CommandOutcome( + label="compile-warm", + exit_code=0, + stdout="{}", + stderr=stderr_by_engine[engine], + ), + ), + compiled={}, + manifest=None, + dag=None, + ) + + return run_engine + + +def first_stat_paths(*, evidence_dir: Path, engines: tuple[str, ...]) -> list[str]: + """The first compiled-test path each engine's stat listing records.""" + + return [ + (evidence_dir / "example__orders" / engine / "compiled-tests-stat.tsv") + .read_text(encoding="utf-8") + .splitlines()[1] + .split("\t")[0] + for engine in engines + ] + + +def kept_files(root: Path) -> tuple[str, ...]: + """Every file below `root` as sorted relative POSIX paths.""" + + return tuple( + sorted(path.relative_to(root).as_posix() for path in filter(Path.is_file, root.rglob("*"))) + ) diff --git a/tests/unit/scripts/compiler_differential/_helpers/running/test_evidence.py b/tests/unit/scripts/compiler_differential/_helpers/running/test_evidence.py new file mode 100644 index 0000000000..d97afea5f4 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/running/test_evidence.py @@ -0,0 +1,94 @@ +"""A differing project keeps both engines' target, stderr and compiled-test stats.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.compiler_differential._helpers.comparing import project_comparison +from scripts.compiler_differential.models import ( + CorpusProject, + DifferentialCommand, + DifferentialOptions, + ExpectedOutcome, + ProjectComparison, +) +from tests.unit.scripts.compiler_differential._helpers.running._test_types import ( + FailureEvidenceTestCase, +) +from tests.unit.scripts.compiler_differential._helpers.running.helpers import ( + fake_run_engine, + first_stat_paths, + kept_files, +) + + +@pytest.mark.parametrize( + "test_case", + [ + FailureEvidenceTestCase( + description="differing warm stderr keeps both engines' evidence", + stderr_by_engine={ + "python": "Semantic checks were partial for 1 models:\n", + "native-preview": "SQL test planning START\n", + }, + expected_files=( + "example__orders/native-preview/0-compile-warm.stderr", + "example__orders/native-preview/compiled-tests-stat.tsv", + "example__orders/native-preview/target/compiled/tests/orders/test_orders.sql", + "example__orders/native-preview/target/sql-test-artifacts.json", + "example__orders/python/0-compile-warm.stderr", + "example__orders/python/compiled-tests-stat.tsv", + "example__orders/python/target/compiled/tests/orders/test_orders.sql", + "example__orders/python/target/sql-test-artifacts.json", + ), + expected_evidence_engines=("native-preview", "python"), + ), + FailureEvidenceTestCase( + description="identical runs keep nothing", + stderr_by_engine={"python": "done\n", "native-preview": "done\n"}, + expected_files=(), + expected_evidence_engines=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_engine_runs_when_comparing_then_only_differing_projects_keep_evidence( + test_case: FailureEvidenceTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + project_comparison, + "run_engine", + fake_run_engine(stderr_by_engine=test_case.stderr_by_engine), + ) + evidence_dir: Path = tmp_path / "evidence" + + comparison: ProjectComparison = project_comparison.compare_project( + project=CorpusProject( + name="example/orders", + commands=(DifferentialCommand(label="compile-warm", arguments=("compile",)),), + expected=ExpectedOutcome(), + source_dir=tmp_path, + ), + options=DifferentialOptions( + engines=("python", "native-preview"), + work_dir=tmp_path / "work", + jobs=1, + stage_captures=False, + python=Path("python"), + engine_environment={}, + evidence_dir=evidence_dir, + ), + ) + stats: list[str] = first_stat_paths( + evidence_dir=evidence_dir, engines=test_case.expected_evidence_engines + ) + + assert (bool(comparison.differences), kept_files(evidence_dir), stats) == ( + bool(test_case.expected_files), + test_case.expected_files, + ["orders/test_orders.sql"] * len(test_case.expected_evidence_engines), + )