ci: lock in compile speed with tighter guards and warm/edit comparisons

.github/workflows/release-performance.yml

@@ -21,6 +21,10 @@ on:
       - .github/workflows/release-performance.yml
       - scripts/compare_release_performance.py
       - scripts/release_performance/**
+      - scripts/compile_performance_ratio/_helpers/edit.py
+      - scripts/compile_performance_ratio/constants.py
+      - scripts/compile_performance_ratio/exceptions.py
+      - scripts/cold_compile_performance/_helpers/dense_project.py
 
 permissions:
   contents: read

.github/workflows/verify.yml

@@ -451,13 +451,14 @@ jobs:
       - name: Install base dependencies
         working-directory: base
         run: uv sync --locked --all-extras
-      - name: Compare base and head cold compiles
+      - name: Compare base and head cold, warm and one-model edit compiles
         env:
           KIND: ${{ matrix.kind }}
           MAX_RATIO: ${{ matrix.max_ratio }}
         run: |
           uv run python -m scripts.compare_compile_performance \
             --kind "${KIND}" \
+            --modes cold warm edit \
             --max-ratio "${MAX_RATIO}" \
             --models 3000 \
             --base-root base \

scripts/compare_compile_performance.py

@@ -6,7 +6,7 @@
 
 
 def main(argv: list[str] | None = None) -> int:
-    """Compare base and head cold compiles on the same machine."""
+    """Compare base and head compiles on the same machine."""
 
     return compare_compile_performance(argv)
 

scripts/compile_performance_ratio/_helpers/edit.py

@@ -0,0 +1,34 @@
+"""Apply a deterministic one-model edit that any generator's benchmark project accepts."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from scripts.compile_performance_ratio.constants import (
+    EDIT_COMMENT,
+    MODEL_FILE_PATTERN,
+    MODELS_DIRECTORY,
+    STATEMENT_START,
+)
+from scripts.compile_performance_ratio.exceptions import BenchmarkEditError
+
+
+def apply_one_model_edit(*, project_dir: Path, revision: int) -> Path:
+    """Comment the middle model's query with the revision so each revision is a new edit."""
+
+    models: list[Path] = sorted(
+        (project_dir / MODELS_DIRECTORY).rglob(MODEL_FILE_PATTERN),
+        key=lambda path: path.relative_to(project_dir).as_posix(),
+    )
+    if not models:
+        raise BenchmarkEditError(f"{project_dir} has no model files to edit")
+    target: Path = models[len(models) // 2]
+    edited, count = STATEMENT_START.subn(
+        EDIT_COMMENT.format(revision=revision) + r"\1",
+        target.read_text(encoding="utf-8"),
+        count=1,
+    )
+    if count != 1:
+        raise BenchmarkEditError(f"{target} has no line starting a WITH or SELECT query to edit")
+    _ = target.write_text(edited, encoding="utf-8")
+    return target

scripts/compile_performance_ratio/_helpers/measure.py

@@ -8,15 +8,20 @@
 import shutil
 import statistics
 import subprocess
+import sys
 import time
 from pathlib import Path
 
 from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project
+from scripts.compile_performance_ratio._helpers.edit import apply_one_model_edit
 from scripts.compile_performance_ratio.constants import (
+    ANALYSIS_CACHE_MISSES,
     BASE_GENERATOR_ENTRY,
     BASE_LABEL,
+    COLD_MODE,
     COMPILE_ENTRY,
     DENSE_KIND,
+    EDIT_MODE,
     ERROR_TAIL_CHARACTERS,
     EXCLUDED_ENVIRONMENT_KEYS,
     EXCLUDED_ENVIRONMENT_PREFIX,
@@ -27,8 +32,10 @@
     FRESH_SOURCE_SHARE,
     FRESH_TEST_SHARE,
     HEAD_LABEL,
+    MODE_TITLES,
     PYTHONPATH_KEY,
     REPORTED_PHASES,
+    WARM_MODE,
 )
 from scripts.compile_performance_ratio.exceptions import CompileComparisonError
 from scripts.compile_performance_ratio.models import CompileComparison, CompileRun
@@ -103,26 +110,57 @@ def compare_builds(
     base_python: Path,
     head_python: Path,
     runs: int,
-) -> CompileComparison:
-    """Alternate base and head cold compiles after one untimed warm-up compile per build."""
+    modes: tuple[str, ...],
+) -> tuple[CompileComparison, ...]:
+    """Alternate base and head compiles per mode; warm and edit reuse the cold project."""
 
     builds: tuple[tuple[str, Path, Path], ...] = (
         (BASE_LABEL, base_python, base_project_dir),
         (HEAD_LABEL, head_python, head_project_dir),
     )
-    for label, python, project_dir in builds:
-        _compile_once(label=label, python=python, project_dir=project_dir)
-    results: dict[str, list[CompileRun]] = {BASE_LABEL: [], HEAD_LABEL: []}
-    for _ in range(runs):
-        for label, python, project_dir in builds:
-            results[label].append(
-                _compile_once(label=label, python=python, project_dir=project_dir)
+    comparisons: list[CompileComparison] = []
+    cache_primed: bool = False
+    for mode in modes:
+        print(f"Comparing {MODE_TITLES[mode]} over {runs} alternating runs", file=sys.stderr)
+        if mode == COLD_MODE:
+            for label, python, project_dir in builds:
+                _ = _compile_once(label=label, python=python, project_dir=project_dir, mode=mode)
+            cache_primed = False
+        elif not cache_primed:
+            for label, python, project_dir in builds:
+                _ = _compile_once(
+                    label=label, python=python, project_dir=project_dir, mode=WARM_MODE
+                )
+            cache_primed = True
+        results: dict[str, list[CompileRun]] = {BASE_LABEL: [], HEAD_LABEL: []}
+        for revision in range(runs):
+            for label, python, project_dir in builds:
+                if mode == EDIT_MODE:
+                    _ = apply_one_model_edit(project_dir=project_dir, revision=revision)
+                run: CompileRun = _compile_once(
+                    label=label, python=python, project_dir=project_dir, mode=mode
+                )
+                _check_cache_use(run=run, mode=mode)
+                results[label].append(run)
+        comparisons.append(
+            _comparison(
+                kind=kind,
+                models=models,
+                mode=mode,
+                base=results[BASE_LABEL],
+                head=results[HEAD_LABEL],
             )
-    base: list[CompileRun] = results[BASE_LABEL]
-    head: list[CompileRun] = results[HEAD_LABEL]
+        )
+    return tuple(comparisons)
+
+
+def _comparison(
+    *, kind: str, models: int, mode: str, base: list[CompileRun], head: list[CompileRun]
+) -> CompileComparison:
     return CompileComparison(
         kind=kind,
         models=models,
+        mode=mode,
         base_wall_seconds=statistics.median(run.wall_seconds for run in base),
         head_wall_seconds=statistics.median(run.wall_seconds for run in head),
         base_cpu_seconds=statistics.median(run.cpu_seconds for run in base),
@@ -132,8 +170,27 @@ def compare_builds(
     )
 
 
-def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun:
-    shutil.rmtree(project_dir / "target", ignore_errors=True)
+def _check_cache_use(*, run: CompileRun, mode: str) -> None:
+    misses: int | None = run.analysis_cache_misses
+    if misses is None or mode == COLD_MODE:
+        return
+    if mode == WARM_MODE and misses != 0:
+        raise CompileComparisonError(
+            f"{run.label} warm compile missed the analysis cache for {misses} models, so it "
+            "did not measure an unchanged warm compile"
+        )
+    if mode == EDIT_MODE and misses == 0:
+        raise CompileComparisonError(
+            f"{run.label} one-model edit compile reported no analysis cache miss, so the edit "
+            "did not invalidate the edited model"
+        )
+
+
+def _compile_once(*, label: str, python: Path, project_dir: Path, mode: str) -> CompileRun:
+    cache_args: tuple[str, ...] = ()
+    if mode == COLD_MODE:
+        shutil.rmtree(project_dir / "target", ignore_errors=True)
+        cache_args = ("--no-cache",)
     environment: dict[str, str] = _compile_environment()
     before: resource.struct_rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
     started: float = time.perf_counter()
@@ -147,7 +204,7 @@ def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun:
             "--no-color",
             "compile",
             "--json",
-            "--no-cache",
+            *cache_args,
         ],
         capture_output=True,
         text=True,
@@ -158,20 +215,22 @@ def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun:
     after: resource.struct_rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
     if completed.returncode != 0:
         raise CompileComparisonError(
-            f"{label} compile failed with exit {completed.returncode}: "
+            f"{label} {mode} compile failed with exit {completed.returncode}: "
             f"{completed.stderr[-ERROR_TAIL_CHARACTERS:]}"
         )
     payload: dict[str, object] = json.loads(completed.stdout)
     timings: object = payload.get("compile_timings", {})
+    numeric: dict[str, int] = {
+        str(name): int(value)
+        for name, value in (timings.items() if isinstance(timings, dict) else ())
+        if isinstance(value, (int, float))
+    }
     return CompileRun(
         label=label,
         wall_seconds=wall_seconds,
         cpu_seconds=(after.ru_utime - before.ru_utime) + (after.ru_stime - before.ru_stime),
-        timings_ms={
-            name: int(value)
-            for name, value in (timings.items() if isinstance(timings, dict) else ())
-            if name in REPORTED_PHASES and isinstance(value, (int, float))
-        },
+        timings_ms={name: value for name, value in numeric.items() if name in REPORTED_PHASES},
+        analysis_cache_misses=numeric.get(ANALYSIS_CACHE_MISSES),
     )
 
 

scripts/compile_performance_ratio/_helpers/report.py

@@ -5,39 +5,54 @@
 import platform
 from pathlib import Path
 
-from scripts.compile_performance_ratio.constants import REPORTED_PHASES
+from scripts.compile_performance_ratio.constants import MODE_TITLES, REPORTED_PHASES
 from scripts.compile_performance_ratio.models import CompileComparison
 
 
 def comparison_markdown(
-    *, comparison: CompileComparison, runs: int, max_ratio: float, per_side_projects: bool
+    *,
+    comparisons: tuple[CompileComparison, ...],
+    runs: int,
+    max_ratio: float,
+    per_side_projects: bool,
+    failures: tuple[str, ...],
 ) -> str:
-    """Summarize medians, ratios and per-phase timings as a Markdown table."""
+    """Summarize medians, ratios and per-phase timings of every mode as Markdown tables."""
 
     generation: str = (
         "Projects generated per side: base with the base checkout's generator, head with head's."
         if per_side_projects
         else "One project generated with head's generator and compiled by both builds."
     )
-    lines: list[str] = [
-        f"### {comparison.kind} {comparison.models} models: head vs base (same runner)",
-        "",
-        f"Runner CPU: {cpu_model()}; {runs} alternating runs each; limit {max_ratio:.2f}x.",
-        "",
-        generation,
-        "",
-        "| Metric | Base | Head | Ratio |",
-        "|---|---:|---:|---:|",
-        f"| Wall (s) | {comparison.base_wall_seconds:.2f} | "
-        f"{comparison.head_wall_seconds:.2f} | {comparison.wall_ratio:.3f} |",
-        f"| CPU (s) | {comparison.base_cpu_seconds:.2f} | "
-        f"{comparison.head_cpu_seconds:.2f} | {comparison.cpu_ratio:.3f} |",
-    ]
-    for phase in REPORTED_PHASES:
-        base: float = comparison.base_timings_ms.get(phase, 0)
-        head: float = comparison.head_timings_ms.get(phase, 0)
-        ratio: str = f"{head / base:.3f}" if base else "n/a"
-        lines.append(f"| {phase} | {base:.0f} | {head:.0f} | {ratio} |")
+    lines: list[str] = []
+    for comparison in comparisons:
+        lines.extend(
+            (
+                f"### {comparison.kind} {comparison.models} models, "
+                f"{MODE_TITLES[comparison.mode]}: head vs base (same runner)",
+                "",
+                f"Runner CPU: {cpu_model()}; {runs} alternating runs each; limit {max_ratio:.2f}x.",
+                "",
+                generation,
+                "",
+                "| Metric | Base | Head | Ratio |",
+                "|---|---:|---:|---:|",
+                f"| Wall (s) | {comparison.base_wall_seconds:.2f} | "
+                f"{comparison.head_wall_seconds:.2f} | {comparison.wall_ratio:.3f} |",
+                f"| CPU (s) | {comparison.base_cpu_seconds:.2f} | "
+                f"{comparison.head_cpu_seconds:.2f} | {comparison.cpu_ratio:.3f} |",
+            )
+        )
+        for phase in REPORTED_PHASES:
+            base: float = comparison.base_timings_ms.get(phase, 0)
+            head: float = comparison.head_timings_ms.get(phase, 0)
+            ratio: str = f"{head / base:.3f}" if base else "n/a"
+            lines.append(f"| {phase} | {base:.0f} | {head:.0f} | {ratio} |")
+        lines.append("")
+    if failures:
+        lines.append("**Result: failed.** " + "; ".join(failures) + ".")
+    else:
+        lines.append("**Result: passed.** Every mode is within the same-runner limit.")
     return "\n".join(lines) + "\n"
 
 

scripts/compile_performance_ratio/_helpers/verdict.py

@@ -0,0 +1,44 @@
+"""Decide whether head compiled slower than base beyond the same-runner ratio limit."""
+
+from __future__ import annotations
+
+import math
+
+from scripts.compile_performance_ratio.models import CompileComparison
+
+
+def ratio_failures(
+    *, comparisons: tuple[CompileComparison, ...], modes: tuple[str, ...], max_ratio: float
+) -> tuple[str, ...]:
+    """Describe every requested mode that is unmeasured or whose wall or CPU ratio is too high."""
+
+    measured: dict[str, CompileComparison] = {
+        comparison.mode: comparison for comparison in comparisons
+    }
+    failures: list[str] = []
+    for mode in modes:
+        comparison: CompileComparison | None = measured.get(mode)
+        if comparison is None:
+            failures.append(f"{mode}: no measurement")
+            continue
+        prefix: str = f"{comparison.kind} {comparison.models} {mode}"
+        for metric, base, head, ratio in (
+            (
+                "wall",
+                comparison.base_wall_seconds,
+                comparison.head_wall_seconds,
+                comparison.wall_ratio,
+            ),
+            ("CPU", comparison.base_cpu_seconds, comparison.head_cpu_seconds, comparison.cpu_ratio),
+        ):
+            if not _measured(base) or not _measured(head):
+                failures.append(
+                    f"{prefix}: {metric} measurement missing (base {base:.2f} s, head {head:.2f} s)"
+                )
+            elif ratio > max_ratio:
+                failures.append(f"{prefix}: {metric} ratio {ratio:.3f} exceeds {max_ratio:.2f}")
+    return tuple(failures)
+
+
+def _measured(seconds: float) -> bool:
+    return math.isfinite(seconds) and seconds > 0

scripts/compile_performance_ratio/constants.py

@@ -1,10 +1,26 @@
 """Stable constants for same-runner compile performance comparisons."""
 
+import re
+
 DEFAULT_RUNS: int = 3
 DEFAULT_MAX_RATIO: float = 1.10
 DENSE_KIND: str = "dense"
 FRESH_KIND: str = "fresh"
 PROJECT_KINDS: tuple[str, ...] = (DENSE_KIND, FRESH_KIND)
+COLD_MODE: str = "cold"
+WARM_MODE: str = "warm"
+EDIT_MODE: str = "edit"
+COMPILE_MODES: tuple[str, ...] = (COLD_MODE, WARM_MODE, EDIT_MODE)
+MODE_TITLES: dict[str, str] = {
+    COLD_MODE: "cold compile without cache",
+    WARM_MODE: "unchanged warm compile",
+    EDIT_MODE: "one-model edit on a warm cache",
+}
+ANALYSIS_CACHE_MISSES: str = "analysis_cache_misses"
+MODELS_DIRECTORY: str = "models"
+MODEL_FILE_PATTERN: str = "*.sql"
+EDIT_COMMENT: str = "-- Benchmark edit {revision}.\n"
+STATEMENT_START: re.Pattern[str] = re.compile(r"^(WITH|SELECT)\b", re.MULTILINE)
 BASE_LABEL: str = "base"
 HEAD_LABEL: str = "head"
 EXCLUDED_ENVIRONMENT_KEYS: frozenset[str] = frozenset({"VIRTUAL_ENV"})

scripts/compile_performance_ratio/exceptions.py

@@ -3,3 +3,7 @@
 
 class CompileComparisonError(RuntimeError):
     """Raised when a compared build fails to compile the benchmark project."""
+
+
+class BenchmarkEditError(RuntimeError):
+    """Raised when a benchmark project has no model query the one-model edit can change."""