88import shutil
99import statistics
1010import subprocess
11+ import sys
1112import time
1213from pathlib import Path
1314
1415from scripts .cold_compile_performance ._helpers .dense_project import write_dense_compile_project
16+ from scripts .compile_performance_ratio ._helpers .edit import apply_one_model_edit
1517from scripts .compile_performance_ratio .constants import (
18+ ANALYSIS_CACHE_MISSES ,
1619 BASE_GENERATOR_ENTRY ,
1720 BASE_LABEL ,
21+ COLD_MODE ,
1822 COMPILE_ENTRY ,
1923 DENSE_KIND ,
24+ EDIT_MODE ,
2025 ERROR_TAIL_CHARACTERS ,
2126 EXCLUDED_ENVIRONMENT_KEYS ,
2227 EXCLUDED_ENVIRONMENT_PREFIX ,
2732 FRESH_SOURCE_SHARE ,
2833 FRESH_TEST_SHARE ,
2934 HEAD_LABEL ,
35+ MODE_TITLES ,
3036 PYTHONPATH_KEY ,
3137 REPORTED_PHASES ,
38+ WARM_MODE ,
3239)
3340from scripts .compile_performance_ratio .exceptions import CompileComparisonError
3441from scripts .compile_performance_ratio .models import CompileComparison , CompileRun
@@ -103,26 +110,57 @@ def compare_builds(
103110 base_python : Path ,
104111 head_python : Path ,
105112 runs : int ,
106- ) -> CompileComparison :
107- """Alternate base and head cold compiles after one untimed warm-up compile per build."""
113+ modes : tuple [str , ...],
114+ ) -> tuple [CompileComparison , ...]:
115+ """Alternate base and head compiles per mode; warm and edit reuse the cold project."""
108116
109117 builds : tuple [tuple [str , Path , Path ], ...] = (
110118 (BASE_LABEL , base_python , base_project_dir ),
111119 (HEAD_LABEL , head_python , head_project_dir ),
112120 )
113- for label , python , project_dir in builds :
114- _compile_once (label = label , python = python , project_dir = project_dir )
115- results : dict [str , list [CompileRun ]] = {BASE_LABEL : [], HEAD_LABEL : []}
116- for _ in range (runs ):
117- for label , python , project_dir in builds :
118- results [label ].append (
119- _compile_once (label = label , python = python , project_dir = project_dir )
121+ comparisons : list [CompileComparison ] = []
122+ cache_primed : bool = False
123+ for mode in modes :
124+ print (f"Comparing { MODE_TITLES [mode ]} over { runs } alternating runs" , file = sys .stderr )
125+ if mode == COLD_MODE :
126+ for label , python , project_dir in builds :
127+ _ = _compile_once (label = label , python = python , project_dir = project_dir , mode = mode )
128+ cache_primed = False
129+ elif not cache_primed :
130+ for label , python , project_dir in builds :
131+ _ = _compile_once (
132+ label = label , python = python , project_dir = project_dir , mode = WARM_MODE
133+ )
134+ cache_primed = True
135+ results : dict [str , list [CompileRun ]] = {BASE_LABEL : [], HEAD_LABEL : []}
136+ for revision in range (runs ):
137+ for label , python , project_dir in builds :
138+ if mode == EDIT_MODE :
139+ _ = apply_one_model_edit (project_dir = project_dir , revision = revision )
140+ run : CompileRun = _compile_once (
141+ label = label , python = python , project_dir = project_dir , mode = mode
142+ )
143+ _check_cache_use (run = run , mode = mode )
144+ results [label ].append (run )
145+ comparisons .append (
146+ _comparison (
147+ kind = kind ,
148+ models = models ,
149+ mode = mode ,
150+ base = results [BASE_LABEL ],
151+ head = results [HEAD_LABEL ],
120152 )
121- base : list [CompileRun ] = results [BASE_LABEL ]
122- head : list [CompileRun ] = results [HEAD_LABEL ]
153+ )
154+ return tuple (comparisons )
155+
156+
157+ def _comparison (
158+ * , kind : str , models : int , mode : str , base : list [CompileRun ], head : list [CompileRun ]
159+ ) -> CompileComparison :
123160 return CompileComparison (
124161 kind = kind ,
125162 models = models ,
163+ mode = mode ,
126164 base_wall_seconds = statistics .median (run .wall_seconds for run in base ),
127165 head_wall_seconds = statistics .median (run .wall_seconds for run in head ),
128166 base_cpu_seconds = statistics .median (run .cpu_seconds for run in base ),
@@ -132,8 +170,27 @@ def compare_builds(
132170 )
133171
134172
135- def _compile_once (* , label : str , python : Path , project_dir : Path ) -> CompileRun :
136- shutil .rmtree (project_dir / "target" , ignore_errors = True )
173+ def _check_cache_use (* , run : CompileRun , mode : str ) -> None :
174+ misses : int | None = run .analysis_cache_misses
175+ if misses is None or mode == COLD_MODE :
176+ return
177+ if mode == WARM_MODE and misses != 0 :
178+ raise CompileComparisonError (
179+ f"{ run .label } warm compile missed the analysis cache for { misses } models, so it "
180+ "did not measure an unchanged warm compile"
181+ )
182+ if mode == EDIT_MODE and misses == 0 :
183+ raise CompileComparisonError (
184+ f"{ run .label } one-model edit compile reported no analysis cache miss, so the edit "
185+ "did not invalidate the edited model"
186+ )
187+
188+
189+ def _compile_once (* , label : str , python : Path , project_dir : Path , mode : str ) -> CompileRun :
190+ cache_args : tuple [str , ...] = ()
191+ if mode == COLD_MODE :
192+ shutil .rmtree (project_dir / "target" , ignore_errors = True )
193+ cache_args = ("--no-cache" ,)
137194 environment : dict [str , str ] = _compile_environment ()
138195 before : resource .struct_rusage = resource .getrusage (resource .RUSAGE_CHILDREN )
139196 started : float = time .perf_counter ()
@@ -147,7 +204,7 @@ def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun:
147204 "--no-color" ,
148205 "compile" ,
149206 "--json" ,
150- "--no-cache" ,
207+ * cache_args ,
151208 ],
152209 capture_output = True ,
153210 text = True ,
@@ -158,20 +215,22 @@ def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun:
158215 after : resource .struct_rusage = resource .getrusage (resource .RUSAGE_CHILDREN )
159216 if completed .returncode != 0 :
160217 raise CompileComparisonError (
161- f"{ label } compile failed with exit { completed .returncode } : "
218+ f"{ label } { mode } compile failed with exit { completed .returncode } : "
162219 f"{ completed .stderr [- ERROR_TAIL_CHARACTERS :]} "
163220 )
164221 payload : dict [str , object ] = json .loads (completed .stdout )
165222 timings : object = payload .get ("compile_timings" , {})
223+ numeric : dict [str , int ] = {
224+ str (name ): int (value )
225+ for name , value in (timings .items () if isinstance (timings , dict ) else ())
226+ if isinstance (value , (int , float ))
227+ }
166228 return CompileRun (
167229 label = label ,
168230 wall_seconds = wall_seconds ,
169231 cpu_seconds = (after .ru_utime - before .ru_utime ) + (after .ru_stime - before .ru_stime ),
170- timings_ms = {
171- name : int (value )
172- for name , value in (timings .items () if isinstance (timings , dict ) else ())
173- if name in REPORTED_PHASES and isinstance (value , (int , float ))
174- },
232+ timings_ms = {name : value for name , value in numeric .items () if name in REPORTED_PHASES },
233+ analysis_cache_misses = numeric .get (ANALYSIS_CACHE_MISSES ),
175234 )
176235
177236
0 commit comments