Skip to content

Commit 110e406

Browse files
authored
ci: lock in compile speed with tighter guards and warm/edit comparisons (#944)
* test: tighten dense compile time and memory budgets to recent CI measurements Cold 1k/3k/5k/10k budgets drop to 13/25/42/77 s and the warm and one-model edit budgets to 11/12 s at 3k and 19/21 s at 5k, about 1.35x the p90 of the dense jobs since the native-analysis parallelism change. Peak RSS limits drop to 800 MiB at 1k, 1.5 GiB at 3k and 2.25 GiB at 5k, about 1.35x the CI maximum; 10k keeps 3.25 GiB. * ci: compare warm and one-model edit compiles against base and the previous release The same-runner ratio job now times cold, unchanged warm and one-model edit compiles of the dense and fresh 3000-model projects for base and head, each on its own generator's project, and fails any mode above 1.10x. Warm and edit reuse the cold project and verify that they hit and miss the analysis cache as intended. The release comparison adds a one-model edit compile of the inspection benchmark and uncached, warm and one-model edit compiles of the dense 3000-model all-rules benchmark, which each version generates with its own generator. All six compile commands are held to a fixed 1.15x wall and CPU limit against the previous release.
1 parent d185100 commit 110e406

36 files changed

Lines changed: 921 additions & 108 deletions

‎.github/workflows/release-performance.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@ on:
2121
- .github/workflows/release-performance.yml
2222
- scripts/compare_release_performance.py
2323
- scripts/release_performance/**
24+
- scripts/compile_performance_ratio/_helpers/edit.py
25+
- scripts/compile_performance_ratio/constants.py
26+
- scripts/compile_performance_ratio/exceptions.py
27+
- scripts/cold_compile_performance/_helpers/dense_project.py
2428

2529
permissions:
2630
contents: read

‎.github/workflows/verify.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -451,13 +451,14 @@ jobs:
451451
- name: Install base dependencies
452452
working-directory: base
453453
run: uv sync --locked --all-extras
454-
- name: Compare base and head cold compiles
454+
- name: Compare base and head cold, warm and one-model edit compiles
455455
env:
456456
KIND: ${{ matrix.kind }}
457457
MAX_RATIO: ${{ matrix.max_ratio }}
458458
run: |
459459
uv run python -m scripts.compare_compile_performance \
460460
--kind "${KIND}" \
461+
--modes cold warm edit \
461462
--max-ratio "${MAX_RATIO}" \
462463
--models 3000 \
463464
--base-root base \

‎scripts/compare_compile_performance.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77

88
def main(argv: list[str] | None = None) -> int:
9-
"""Compare base and head cold compiles on the same machine."""
9+
"""Compare base and head compiles on the same machine."""
1010

1111
return compare_compile_performance(argv)
1212

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
"""Apply a deterministic one-model edit that any generator's benchmark project accepts."""
2+
3+
from __future__ import annotations
4+
5+
from pathlib import Path
6+
7+
from scripts.compile_performance_ratio.constants import (
8+
EDIT_COMMENT,
9+
MODEL_FILE_PATTERN,
10+
MODELS_DIRECTORY,
11+
STATEMENT_START,
12+
)
13+
from scripts.compile_performance_ratio.exceptions import BenchmarkEditError
14+
15+
16+
def apply_one_model_edit(*, project_dir: Path, revision: int) -> Path:
17+
"""Comment the middle model's query with the revision so each revision is a new edit."""
18+
19+
models: list[Path] = sorted(
20+
(project_dir / MODELS_DIRECTORY).rglob(MODEL_FILE_PATTERN),
21+
key=lambda path: path.relative_to(project_dir).as_posix(),
22+
)
23+
if not models:
24+
raise BenchmarkEditError(f"{project_dir} has no model files to edit")
25+
target: Path = models[len(models) // 2]
26+
edited, count = STATEMENT_START.subn(
27+
EDIT_COMMENT.format(revision=revision) + r"\1",
28+
target.read_text(encoding="utf-8"),
29+
count=1,
30+
)
31+
if count != 1:
32+
raise BenchmarkEditError(f"{target} has no line starting a WITH or SELECT query to edit")
33+
_ = target.write_text(edited, encoding="utf-8")
34+
return target

‎scripts/compile_performance_ratio/_helpers/measure.py‎

Lines changed: 79 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,20 @@
88
import shutil
99
import statistics
1010
import subprocess
11+
import sys
1112
import time
1213
from pathlib import Path
1314

1415
from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project
16+
from scripts.compile_performance_ratio._helpers.edit import apply_one_model_edit
1517
from scripts.compile_performance_ratio.constants import (
18+
ANALYSIS_CACHE_MISSES,
1619
BASE_GENERATOR_ENTRY,
1720
BASE_LABEL,
21+
COLD_MODE,
1822
COMPILE_ENTRY,
1923
DENSE_KIND,
24+
EDIT_MODE,
2025
ERROR_TAIL_CHARACTERS,
2126
EXCLUDED_ENVIRONMENT_KEYS,
2227
EXCLUDED_ENVIRONMENT_PREFIX,
@@ -27,8 +32,10 @@
2732
FRESH_SOURCE_SHARE,
2833
FRESH_TEST_SHARE,
2934
HEAD_LABEL,
35+
MODE_TITLES,
3036
PYTHONPATH_KEY,
3137
REPORTED_PHASES,
38+
WARM_MODE,
3239
)
3340
from scripts.compile_performance_ratio.exceptions import CompileComparisonError
3441
from scripts.compile_performance_ratio.models import CompileComparison, CompileRun
@@ -103,26 +110,57 @@ def compare_builds(
103110
base_python: Path,
104111
head_python: Path,
105112
runs: int,
106-
) -> CompileComparison:
107-
"""Alternate base and head cold compiles after one untimed warm-up compile per build."""
113+
modes: tuple[str, ...],
114+
) -> tuple[CompileComparison, ...]:
115+
"""Alternate base and head compiles per mode; warm and edit reuse the cold project."""
108116

109117
builds: tuple[tuple[str, Path, Path], ...] = (
110118
(BASE_LABEL, base_python, base_project_dir),
111119
(HEAD_LABEL, head_python, head_project_dir),
112120
)
113-
for label, python, project_dir in builds:
114-
_compile_once(label=label, python=python, project_dir=project_dir)
115-
results: dict[str, list[CompileRun]] = {BASE_LABEL: [], HEAD_LABEL: []}
116-
for _ in range(runs):
117-
for label, python, project_dir in builds:
118-
results[label].append(
119-
_compile_once(label=label, python=python, project_dir=project_dir)
121+
comparisons: list[CompileComparison] = []
122+
cache_primed: bool = False
123+
for mode in modes:
124+
print(f"Comparing {MODE_TITLES[mode]} over {runs} alternating runs", file=sys.stderr)
125+
if mode == COLD_MODE:
126+
for label, python, project_dir in builds:
127+
_ = _compile_once(label=label, python=python, project_dir=project_dir, mode=mode)
128+
cache_primed = False
129+
elif not cache_primed:
130+
for label, python, project_dir in builds:
131+
_ = _compile_once(
132+
label=label, python=python, project_dir=project_dir, mode=WARM_MODE
133+
)
134+
cache_primed = True
135+
results: dict[str, list[CompileRun]] = {BASE_LABEL: [], HEAD_LABEL: []}
136+
for revision in range(runs):
137+
for label, python, project_dir in builds:
138+
if mode == EDIT_MODE:
139+
_ = apply_one_model_edit(project_dir=project_dir, revision=revision)
140+
run: CompileRun = _compile_once(
141+
label=label, python=python, project_dir=project_dir, mode=mode
142+
)
143+
_check_cache_use(run=run, mode=mode)
144+
results[label].append(run)
145+
comparisons.append(
146+
_comparison(
147+
kind=kind,
148+
models=models,
149+
mode=mode,
150+
base=results[BASE_LABEL],
151+
head=results[HEAD_LABEL],
120152
)
121-
base: list[CompileRun] = results[BASE_LABEL]
122-
head: list[CompileRun] = results[HEAD_LABEL]
153+
)
154+
return tuple(comparisons)
155+
156+
157+
def _comparison(
158+
*, kind: str, models: int, mode: str, base: list[CompileRun], head: list[CompileRun]
159+
) -> CompileComparison:
123160
return CompileComparison(
124161
kind=kind,
125162
models=models,
163+
mode=mode,
126164
base_wall_seconds=statistics.median(run.wall_seconds for run in base),
127165
head_wall_seconds=statistics.median(run.wall_seconds for run in head),
128166
base_cpu_seconds=statistics.median(run.cpu_seconds for run in base),
@@ -132,8 +170,27 @@ def compare_builds(
132170
)
133171

134172

135-
def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun:
136-
shutil.rmtree(project_dir / "target", ignore_errors=True)
173+
def _check_cache_use(*, run: CompileRun, mode: str) -> None:
174+
misses: int | None = run.analysis_cache_misses
175+
if misses is None or mode == COLD_MODE:
176+
return
177+
if mode == WARM_MODE and misses != 0:
178+
raise CompileComparisonError(
179+
f"{run.label} warm compile missed the analysis cache for {misses} models, so it "
180+
"did not measure an unchanged warm compile"
181+
)
182+
if mode == EDIT_MODE and misses == 0:
183+
raise CompileComparisonError(
184+
f"{run.label} one-model edit compile reported no analysis cache miss, so the edit "
185+
"did not invalidate the edited model"
186+
)
187+
188+
189+
def _compile_once(*, label: str, python: Path, project_dir: Path, mode: str) -> CompileRun:
190+
cache_args: tuple[str, ...] = ()
191+
if mode == COLD_MODE:
192+
shutil.rmtree(project_dir / "target", ignore_errors=True)
193+
cache_args = ("--no-cache",)
137194
environment: dict[str, str] = _compile_environment()
138195
before: resource.struct_rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
139196
started: float = time.perf_counter()
@@ -147,7 +204,7 @@ def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun:
147204
"--no-color",
148205
"compile",
149206
"--json",
150-
"--no-cache",
207+
*cache_args,
151208
],
152209
capture_output=True,
153210
text=True,
@@ -158,20 +215,22 @@ def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun:
158215
after: resource.struct_rusage = resource.getrusage(resource.RUSAGE_CHILDREN)
159216
if completed.returncode != 0:
160217
raise CompileComparisonError(
161-
f"{label} compile failed with exit {completed.returncode}: "
218+
f"{label} {mode} compile failed with exit {completed.returncode}: "
162219
f"{completed.stderr[-ERROR_TAIL_CHARACTERS:]}"
163220
)
164221
payload: dict[str, object] = json.loads(completed.stdout)
165222
timings: object = payload.get("compile_timings", {})
223+
numeric: dict[str, int] = {
224+
str(name): int(value)
225+
for name, value in (timings.items() if isinstance(timings, dict) else ())
226+
if isinstance(value, (int, float))
227+
}
166228
return CompileRun(
167229
label=label,
168230
wall_seconds=wall_seconds,
169231
cpu_seconds=(after.ru_utime - before.ru_utime) + (after.ru_stime - before.ru_stime),
170-
timings_ms={
171-
name: int(value)
172-
for name, value in (timings.items() if isinstance(timings, dict) else ())
173-
if name in REPORTED_PHASES and isinstance(value, (int, float))
174-
},
232+
timings_ms={name: value for name, value in numeric.items() if name in REPORTED_PHASES},
233+
analysis_cache_misses=numeric.get(ANALYSIS_CACHE_MISSES),
175234
)
176235

177236

‎scripts/compile_performance_ratio/_helpers/report.py‎

Lines changed: 37 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,39 +5,54 @@
55
import platform
66
from pathlib import Path
77

8-
from scripts.compile_performance_ratio.constants import REPORTED_PHASES
8+
from scripts.compile_performance_ratio.constants import MODE_TITLES, REPORTED_PHASES
99
from scripts.compile_performance_ratio.models import CompileComparison
1010

1111

1212
def comparison_markdown(
13-
*, comparison: CompileComparison, runs: int, max_ratio: float, per_side_projects: bool
13+
*,
14+
comparisons: tuple[CompileComparison, ...],
15+
runs: int,
16+
max_ratio: float,
17+
per_side_projects: bool,
18+
failures: tuple[str, ...],
1419
) -> str:
15-
"""Summarize medians, ratios and per-phase timings as a Markdown table."""
20+
"""Summarize medians, ratios and per-phase timings of every mode as Markdown tables."""
1621

1722
generation: str = (
1823
"Projects generated per side: base with the base checkout's generator, head with head's."
1924
if per_side_projects
2025
else "One project generated with head's generator and compiled by both builds."
2126
)
22-
lines: list[str] = [
23-
f"### {comparison.kind} {comparison.models} models: head vs base (same runner)",
24-
"",
25-
f"Runner CPU: {cpu_model()}; {runs} alternating runs each; limit {max_ratio:.2f}x.",
26-
"",
27-
generation,
28-
"",
29-
"| Metric | Base | Head | Ratio |",
30-
"|---|---:|---:|---:|",
31-
f"| Wall (s) | {comparison.base_wall_seconds:.2f} | "
32-
f"{comparison.head_wall_seconds:.2f} | {comparison.wall_ratio:.3f} |",
33-
f"| CPU (s) | {comparison.base_cpu_seconds:.2f} | "
34-
f"{comparison.head_cpu_seconds:.2f} | {comparison.cpu_ratio:.3f} |",
35-
]
36-
for phase in REPORTED_PHASES:
37-
base: float = comparison.base_timings_ms.get(phase, 0)
38-
head: float = comparison.head_timings_ms.get(phase, 0)
39-
ratio: str = f"{head / base:.3f}" if base else "n/a"
40-
lines.append(f"| {phase} | {base:.0f} | {head:.0f} | {ratio} |")
27+
lines: list[str] = []
28+
for comparison in comparisons:
29+
lines.extend(
30+
(
31+
f"### {comparison.kind} {comparison.models} models, "
32+
f"{MODE_TITLES[comparison.mode]}: head vs base (same runner)",
33+
"",
34+
f"Runner CPU: {cpu_model()}; {runs} alternating runs each; limit {max_ratio:.2f}x.",
35+
"",
36+
generation,
37+
"",
38+
"| Metric | Base | Head | Ratio |",
39+
"|---|---:|---:|---:|",
40+
f"| Wall (s) | {comparison.base_wall_seconds:.2f} | "
41+
f"{comparison.head_wall_seconds:.2f} | {comparison.wall_ratio:.3f} |",
42+
f"| CPU (s) | {comparison.base_cpu_seconds:.2f} | "
43+
f"{comparison.head_cpu_seconds:.2f} | {comparison.cpu_ratio:.3f} |",
44+
)
45+
)
46+
for phase in REPORTED_PHASES:
47+
base: float = comparison.base_timings_ms.get(phase, 0)
48+
head: float = comparison.head_timings_ms.get(phase, 0)
49+
ratio: str = f"{head / base:.3f}" if base else "n/a"
50+
lines.append(f"| {phase} | {base:.0f} | {head:.0f} | {ratio} |")
51+
lines.append("")
52+
if failures:
53+
lines.append("**Result: failed.** " + "; ".join(failures) + ".")
54+
else:
55+
lines.append("**Result: passed.** Every mode is within the same-runner limit.")
4156
return "\n".join(lines) + "\n"
4257

4358

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
"""Decide whether head compiled slower than base beyond the same-runner ratio limit."""
2+
3+
from __future__ import annotations
4+
5+
import math
6+
7+
from scripts.compile_performance_ratio.models import CompileComparison
8+
9+
10+
def ratio_failures(
11+
*, comparisons: tuple[CompileComparison, ...], modes: tuple[str, ...], max_ratio: float
12+
) -> tuple[str, ...]:
13+
"""Describe every requested mode that is unmeasured or whose wall or CPU ratio is too high."""
14+
15+
measured: dict[str, CompileComparison] = {
16+
comparison.mode: comparison for comparison in comparisons
17+
}
18+
failures: list[str] = []
19+
for mode in modes:
20+
comparison: CompileComparison | None = measured.get(mode)
21+
if comparison is None:
22+
failures.append(f"{mode}: no measurement")
23+
continue
24+
prefix: str = f"{comparison.kind} {comparison.models} {mode}"
25+
for metric, base, head, ratio in (
26+
(
27+
"wall",
28+
comparison.base_wall_seconds,
29+
comparison.head_wall_seconds,
30+
comparison.wall_ratio,
31+
),
32+
("CPU", comparison.base_cpu_seconds, comparison.head_cpu_seconds, comparison.cpu_ratio),
33+
):
34+
if not _measured(base) or not _measured(head):
35+
failures.append(
36+
f"{prefix}: {metric} measurement missing (base {base:.2f} s, head {head:.2f} s)"
37+
)
38+
elif ratio > max_ratio:
39+
failures.append(f"{prefix}: {metric} ratio {ratio:.3f} exceeds {max_ratio:.2f}")
40+
return tuple(failures)
41+
42+
43+
def _measured(seconds: float) -> bool:
44+
return math.isfinite(seconds) and seconds > 0

‎scripts/compile_performance_ratio/constants.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,26 @@
11
"""Stable constants for same-runner compile performance comparisons."""
22

3+
import re
4+
35
DEFAULT_RUNS: int = 3
46
DEFAULT_MAX_RATIO: float = 1.10
57
DENSE_KIND: str = "dense"
68
FRESH_KIND: str = "fresh"
79
PROJECT_KINDS: tuple[str, ...] = (DENSE_KIND, FRESH_KIND)
10+
COLD_MODE: str = "cold"
11+
WARM_MODE: str = "warm"
12+
EDIT_MODE: str = "edit"
13+
COMPILE_MODES: tuple[str, ...] = (COLD_MODE, WARM_MODE, EDIT_MODE)
14+
MODE_TITLES: dict[str, str] = {
15+
COLD_MODE: "cold compile without cache",
16+
WARM_MODE: "unchanged warm compile",
17+
EDIT_MODE: "one-model edit on a warm cache",
18+
}
19+
ANALYSIS_CACHE_MISSES: str = "analysis_cache_misses"
20+
MODELS_DIRECTORY: str = "models"
21+
MODEL_FILE_PATTERN: str = "*.sql"
22+
EDIT_COMMENT: str = "-- Benchmark edit {revision}.\n"
23+
STATEMENT_START: re.Pattern[str] = re.compile(r"^(WITH|SELECT)\b", re.MULTILINE)
824
BASE_LABEL: str = "base"
925
HEAD_LABEL: str = "head"
1026
EXCLUDED_ENVIRONMENT_KEYS: frozenset[str] = frozenset({"VIRTUAL_ENV"})

‎scripts/compile_performance_ratio/exceptions.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,7 @@
33

44
class CompileComparisonError(RuntimeError):
55
"""Raised when a compared build fails to compile the benchmark project."""
6+
7+
8+
class BenchmarkEditError(RuntimeError):
9+
"""Raised when a benchmark project has no model query the one-model edit can change."""

0 commit comments

Comments
 (0)