Skip to content

Commit 68a50d3

Browse files
ci: Fix benchmarks (#591)
* bench: fetch-main reads report results via per-uuid GET Bencher's reports LIST endpoint stopped inlining each report's `results` (it now carries only a `counts` summary), which fetch-main read as "report found, zero rows" — every !benchmark base fetch on every backend then exited 3 and silently fell back to a FULL base-side rerun (rebuild the base binaries + re-measure), ~5 min of redundant work per entry despite the rows sitting on bencher. Fetch each SHA-matched report by uuid (the single-report GET still returns full results) before row parsing; an inlined results array is still used directly if the API populates it again. Validated against live bencher at main@7da4132: the aiur-sharded-env/ISLB fetch that drove the fallback in run 32749751301 now returns its row (5 measures, empty missing list, exit 0), and a compile fetch at the same SHA exercises the multi-report-per-testbed aggregation. * bench: display titles for the aiur-sharded-env-check plots The five new plots synced with the slug fallback ("aiur-sharded-env- check: check-time"); name them in the house style — Aiur Env Check Time / Throughput / Peak RAM Usage, Aiur Env Constants, Aiur Env Shards. Verified via `ix bench plots --dry-run`. * ci: lean-update sweeps every Benchmarks package via the /** glob `. Benchmarks/**` replaces the hand-listed ". Benchmarks/Compile": the recursive form reaches all seven packages under Benchmarks/ (Catalog, CatalogReal, CatalogSpine, Compile, CompileFC, and Catalog's nested RelocFixture workspaces) while skipping dotted directories, so .lake dependency checkouts stay untouched. CompileFC was previously left pinned to an old toolchain on purpose; it now updates with the rest.
1 parent e47a1b7 commit 68a50d3

3 files changed

Lines changed: 33 additions & 4 deletions

File tree

.github/workflows/update.yml

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,12 @@ jobs:
3232
# pinned to a commit hash is reported and left alone.
3333
- uses: argumentcomputer/lean-update@dev
3434
with:
35-
# The root package and the compile benchmarks; Benchmarks/CompileFC
36-
# is deliberately left on its old toolchain, so no glob here.
37-
lake_package_directory: ". Benchmarks/Compile"
35+
# The root package plus every package under Benchmarks/ — `/**`
36+
# walks the whole tree (catching Catalog's nested fixture
37+
# workspaces) and skips dotted directories, so `.lake`
38+
# dependency checkouts are never swept up. This includes
39+
# Benchmarks/CompileFC, previously pinned to an old toolchain.
40+
lake_package_directory: ". Benchmarks/**"
3841
bump_mode: pinned-tags
3942
pr: true
4043
token: ${{ steps.app-token.outputs.token }}

Ix/Cli/BenchPlots.lean

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,11 @@ def plotTitle (workload measure : String) : String :=
7878
| "ooc-check", "check-time" => "OOC Check Time"
7979
| "ooc-check", "throughput" => "OOC Check Throughput"
8080
| "ooc-check", "peak-rss" => "OOC Check Peak RAM Usage"
81+
| "aiur-sharded-env-check", "check-time" => "Aiur Env Check Time"
82+
| "aiur-sharded-env-check", "throughput" => "Aiur Env Check Throughput"
83+
| "aiur-sharded-env-check", "peak-rss" => "Aiur Env Check Peak RAM Usage"
84+
| "aiur-sharded-env-check", "constants" => "Aiur Env Constants"
85+
| "aiur-sharded-env-check", "shards" => "Aiur Env Shards"
8186
| w, m => s!"{w}: {m}"
8287

8388
/-- Tracked but not plotted solo. Zisk

Ix/Cli/BenchReport.lean

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -822,9 +822,30 @@ def runFetchMainCmd (p : Cli.Parsed) : IO UInt32 := do
822822
IO.println s!"fetch-main: no reports for {backend}/{mode} @ {sha.take 8}"
823823
return exitRejected
824824

825+
-- The LIST endpoint stopped inlining each report's `results` (it now
826+
-- carries only a `counts` summary), which read here as "report found,
827+
-- zero rows" and sent every caller down the full base-rerun fallback.
828+
-- Fetch each matched report by uuid — the single-report GET still
829+
-- returns full results; the inline array is used when a future API
830+
-- (or cached response) populates it again.
831+
let mut detailed : Array Json := #[]
832+
for r in atSha do
833+
let inline := (r.getObjVal? "results").toOption.bind (·.getArr?.toOption)
834+
|>.getD #[]
835+
if !inline.isEmpty then
836+
detailed := detailed.push r
837+
else
838+
let some uuid := (r.getObjVal? "uuid").toOption.bind (·.getStr?.toOption)
839+
| continue
840+
match ← curlJson s!"https://api.bencher.dev/v0/projects/ix/reports/{uuid}" with
841+
| .error e =>
842+
IO.println s!"fetch-main: bencher API error (report {uuid}): {e}"
843+
return exitRejected
844+
| .ok full => detailed := detailed.push full
845+
825846
let mut rows : Array (String × Json) := #[]
826847
let mut seen : Array String := #[]
827-
for r in atSha do
848+
for r in detailed do
828849
let iterations := (r.getObjVal? "results").toOption.bind (·.getArr?.toOption)
829850
|>.getD #[]
830851
for iteration in iterations do

0 commit comments

Comments
 (0)