Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 48 additions & 8 deletions .github/workflows/benchmark-application.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,18 +39,24 @@ jobs:
- name: Build application and test checkpoint/ZIP safeguards
run: mvn -B -f benchmark/pom.xml package
- name: Verify automatic preparation and all library/profile adapters
run: java -Xmx2g -jar benchmark/target/nmdev-benchmark-1.3.0.jar --smoke --output smoke-results
run: java -Xmx2g -jar benchmark/target/nmdev-benchmark-1.3.1.jar --smoke --output smoke-results
- name: Verify arbitrary local NMDEV filename and recorded binary origin
shell: pwsh
run: |
Copy-Item smoke-results/.runtime/lib/nmdev-selected.jar 'selected NMDEV.jar'
java -Xmx2g -jar benchmark/target/nmdev-benchmark-1.3.0.jar --nmdev-jar 'selected NMDEV.jar' --smoke --libraries 'NMDEV,COMMONS,COMMONS_BLOCK' --max-cases 1 --output local-jar-results
java -Xmx2g -jar benchmark/target/nmdev-benchmark-1.3.1.jar --nmdev-jar 'selected NMDEV.jar' --smoke --libraries 'NMDEV,COMMONS,COMMONS_BLOCK' --max-cases 1 --output local-jar-results
if ($LASTEXITCODE -ne 0) { throw 'Local-JAR smoke failed' }
$expected = (Get-FileHash 'selected NMDEV.jar' -Algorithm SHA256).Hash.ToLowerInvariant()
$recorded = Get-Content local-jar-results/binary-provenance.json -Raw | ConvertFrom-Json
if ($recorded.nmdev_sha256 -ne $expected -or $recorded.nmdev_original_filename -ne 'selected NMDEV.jar') { throw 'Wrong NMDEV binary provenance' }
- name: Verify saved forks are reused
run: java -Xmx2g -jar benchmark/target/nmdev-benchmark-1.3.0.jar --smoke --output smoke-results
- name: Verify fast resume does not iterate completed cases
shell: pwsh
run: |
$resumed = java -Xmx2g -jar benchmark/target/nmdev-benchmark-1.3.1.jar --smoke --output smoke-results
$resumed | Write-Host
if ($LASTEXITCODE -ne 0) { throw 'Resume failed' }
if ($resumed -match '^Case [0-9]+/') { throw 'Resume revisited completed case loop' }
if (-not ($resumed -match '^Resume: verified [1-9]')) { throw 'No completed checkpoints restored' }
- uses: actions/upload-artifact@v4
if: always()
with:
Expand All @@ -61,11 +67,45 @@ jobs:
if: matrix.os == 'ubuntu-24.04' && success()
with:
name: nmdev-benchmark-application
path: benchmark/target/nmdev-benchmark-1.3.0.jar
path: benchmark/target/nmdev-benchmark-1.3.1.jar

upgrade-published:
needs: portable
runs-on: windows-2022
timeout-minutes: 15
steps:
- uses: actions/setup-java@v4
with:
distribution: temurin
java-version: '17'
- uses: actions/download-artifact@v4
with:
name: nmdev-benchmark-application
path: release
- name: Verify published 1.3.0 run upgrades without changing saved forks
shell: pwsh
run: |
Invoke-WebRequest -UseBasicParsing 'https://github.com/nmltd/nmdev-examples/releases/download/benchmark-v1.3.0/nmdev-benchmark-1.3.0.jar' -OutFile original-1.3.0.jar
if ((Get-FileHash original-1.3.0.jar -Algorithm SHA256).Hash.ToLowerInvariant() -ne 'a6b58052fedb17ea91f4c2812ac816708c700dfc386a9e3019306e9a26fde9cf') { throw 'Original binary hash mismatch' }
New-Item -ItemType Directory upgrade-results | Out-Null
java -Xmx2g -jar original-1.3.0.jar --smoke --libraries 'NMDEV,COMMONS,COMMONS_BLOCK' --max-cases 1 --output upgrade-results
if ($LASTEXITCODE -ne 0) { throw 'Original run failed' }
$before = @{}
Get-ChildItem upgrade-results/cases -Recurse -File | Where-Object { $_.Name -like 'fork-*' -and $_.Extension -ne '.log' } | ForEach-Object { $before[$_.FullName] = (Get-FileHash $_.FullName -Algorithm SHA256).Hash }
# Simulate a crash after both forks were saved but before the case report was exported.
Get-ChildItem upgrade-results/cases -Recurse -Filter result.json | Remove-Item
java -cp release/nmdev-benchmark-1.3.1.jar dev.nm.benchmark.launcher.ResumeUpgrade original-1.3.0.jar upgrade-results
if ($LASTEXITCODE -ne 0) { throw 'Upgrade verification failed' }
$resumed = java -Xmx2g -jar release/nmdev-benchmark-1.3.1.jar --smoke --libraries 'NMDEV,COMMONS,COMMONS_BLOCK' --max-cases 1 --output upgrade-results
$resumed | Write-Host
if ($LASTEXITCODE -ne 0 -or ($resumed -match '^Case [0-9]+/')) { throw 'Upgraded run repeated a completed case' }
if (@(Get-ChildItem upgrade-results/cases -Recurse -Filter result.json).Count -ne 1) { throw 'Missing case report was not rebuilt' }
foreach ($path in $before.Keys) { if ((Get-FileHash $path -Algorithm SHA256).Hash -ne $before[$path]) { throw 'Saved fork changed during upgrade/resume' } }


release:
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
needs: portable
needs: [portable, upgrade-published]
runs-on: ubuntu-latest
permissions:
contents: write
Expand All @@ -80,5 +120,5 @@ jobs:
GH_TOKEN: ${{ github.token }}
run: |
cd release
sha256sum nmdev-benchmark-1.3.0.jar > SHA256SUMS
gh release create benchmark-v1.3.0 nmdev-benchmark-1.3.0.jar SHA256SUMS ../benchmark/run-benchmark.ps1 --repo nmltd/nmdev-examples --target "$GITHUB_SHA" --title 'Reusable benchmark application 1.3.0' --notes 'Select any NMDEV binary with --nmdev-jar. Full 1,260-case suite across eleven libraries including both Commons matrix implementations. Released checksum-pinned Commons Math 4.0-beta1 dependencies. Windows and Linux application smoke, local-JAR selection and resume checks passed for this exact application build. Full benchmark timings are run locally.'
sha256sum nmdev-benchmark-1.3.1.jar > SHA256SUMS
gh release create benchmark-v1.3.1 nmdev-benchmark-1.3.1.jar SHA256SUMS ../benchmark/run-benchmark.ps1 --repo nmltd/nmdev-examples --target "$GITHUB_SHA" --title 'Faster benchmark resume 1.3.1' --notes 'Select any NMDEV binary with --nmdev-jar. Full 1,260-case suite across eleven libraries including both Commons matrix implementations. Released checksum-pinned Commons Math 4.0-beta1 dependencies. Windows and Linux application smoke, local-JAR selection and resume checks passed for this exact application build. Fast checkpoint scan and periodic ZIP exports. Published 1.3.0 resume migration verified with unchanged saved fork hashes. Full benchmark timings are run locally.'
14 changes: 14 additions & 0 deletions benchmark/LOCAL-RUN.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,17 @@
# Faster resume and exports (application 1.3.1)

Completed cases are verified in one scan and collected without re-entering the measurement loop. Reports are rebuilt once before unfinished work; fully completed resumes only export at the end. Fork checkpoints are still written and synchronized immediately. During measurement, full ZIP snapshots are limited to once a minute; orderly completion and interruption force a final snapshot. Brief Windows AccessDenied failures during file replacement are retried without deleting the previous file or ignoring checkpoint errors.

The checked-in PowerShell entry point downloads application 1.3.1. Existing 1.3.0 result directories continue using their original runner unless an explicit verified resume upgrade is requested. To upgrade a stopped published 1.3.0 run:

```powershell
powershell -ExecutionPolicy Bypass -File run-benchmark.ps1 -NmdevVersion 1.2.10 -UpgradeResume
```

Stop an active runner with Ctrl+C first. The helper verifies the original published 1.3.0 SHA-256 and byte-compares the application JARs: numerical harness sources, dependencies, case-model class and other application classes must match. Only the audited resume/export classes, added upgrade helper and application version metadata may differ. It locks the result directory, backs up request.json, records provenance and updates only the application hash. Saved fork files, measurements, selected NMDEV binary and case specifications are not changed. Full normal machine/configuration/dependency and fork integrity checks run after migration. An unknown original build or any unexpected JAR entry difference is rejected.

---

# Reusable benchmark application 1.3.0

From PowerShell, run `benchmark/run-benchmark.ps1`. It downloads the checksum-verified prebuilt runner and NMDEV 1.2.10, then starts the complete suite. A full JDK 17+ is required; Maven and source edits are unnecessary.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ public static void main(String[] args) throws Exception {
case "--smoke": smoke=true; break;
case "--max-cases": maxCases=Integer.parseInt(args[++i]); break;
case "--libraries": libraries=List.of(args[++i].split(",")); if(!LIBRARIES.containsAll(libraries)) throw new IllegalArgumentException("Unknown library"); break;
case "--help": System.out.println("java -jar nmdev-benchmark-1.3.0.jar [--nmdev-jar path/to/nmdev.jar] [--output directory]\nNo arguments runs all 11 libraries (12 implementation variants) and all profiles. Repeat the same command to resume.\nDiagnostics only: --smoke [--libraries NMDEV,EJML] [--max-cases N]"); return;
case "--help": System.out.println("java -jar nmdev-benchmark-1.3.1.jar [--nmdev-jar path/to/nmdev.jar] [--output directory]\nNo arguments runs all 11 libraries (12 implementation variants) and all profiles. Repeat the same command to resume.\nDiagnostics only: --smoke [--libraries NMDEV,EJML] [--max-cases N]"); return;
default: throw new IllegalArgumentException("Unknown argument: "+args[i]);
}
if (!smoke && !libraries.equals(LIBRARIES)) throw new IllegalArgumentException("--libraries is a diagnostic option requiring --smoke");
Expand Down Expand Up @@ -99,7 +99,7 @@ static void run(Path output, boolean smoke, List<String> libraries, int maxCases
request.put("child_jvm_options","-Xms256m -Xmx1g; ActiveProcessorCount and benchmark.threads per case; F2J BLAS/LAPACK/ARPACK forced");
Path manifest=output.resolve("request.json"); JsonElement specification=Storage.JSON.toJsonTree(request);
if(Files.exists(manifest) && !Storage.read(manifest).equals(specification)) throw new IOException("Run configuration, binaries or machine changed. Use a different --output directory; existing checkpoints were preserved.");
Storage.json(manifest,specification); Files.deleteIfExists(output.resolve("completed-utc.txt"));
Storage.json(manifest,specification);
Storage.atomic(output.resolve("README.md"),("# NM Dev benchmark output\n\nSuite: "+request.get("suite")+". Profiles ending in -all-cores request all JVM-visible logical processors recorded in request.json. Single-threaded APIs remain single threaded; requested workers do not establish actual utilization. COMMONS uses Array2DRowRealMatrix; COMMONS_BLOCK uses BlockRealMatrix, constructed before timing. Compare matching operations and dimensions across profiles; do not pool thread settings. All measurements are local to the machine recorded in machine.json.\n\nCase checkpoints include two independently warmed JVM forks. Fork JSON is preserved verbatim; combined JSON uses JMH's own aggregators. Requested native thread counts are not independently audited.\n\nUpload the sibling *-results-<UTC timestamp>.zip. It excludes licenses, downloaded binaries, credentials and private runtime cache. Partial runs are explicitly marked in progress.json. To resume, keep the complete local output directory, including .runtime and binary fork checkpoints, and run the same JAR again.\n").getBytes(java.nio.charset.StandardCharsets.UTF_8));
// Preflight failures are recorded; each timed fork independently validates its
// own full-size inputs. An unsupported native backend does not hide other libraries.
Expand All @@ -113,10 +113,21 @@ static void run(Path output, boolean smoke, List<String> libraries, int maxCases
}
if(Storage.read(done).getAsJsonObject().get("exit_code").getAsInt()!=0) preflightFailures.add(name);
}
JsonArray combined=new JsonArray(); List<Map<String,Object>> failed=new ArrayList<>(); int completed=0,attempted=0;
List<Case> selected=cases.subList(0,Math.min(Math.max(maxCases,0),cases.size()));
Map<String,JsonObject> results=restoreCompleted(output,selected);
List<Case> pendingCases=selected.stream().filter(item->!results.containsKey(item.id())).toList();
JsonArray combined=orderedResults(selected,results);
List<Map<String,Object>> failed=new ArrayList<>(); int completed=results.size(),attempted=0;
System.out.println("Resume: verified "+completed+" completed cases; "+pendingCases.size()+" cases need work in this invocation.");
if(!pendingCases.isEmpty() || !preflightFailures.isEmpty()) Files.deleteIfExists(output.resolve("completed-utc.txt"));
// Rebuild saved reports once, not once for each retained case.
if(completed>0 && !pendingCases.isEmpty()) {
Storage.jsonIfChanged(output.resolve("jmh.json"),combined); summary(output,combined);
}
boolean measurementsFinished=false;
try {
for(Case item:cases) {
if(attempted++>=maxCases) break;
for(Case item:pendingCases) {
attempted=cases.indexOf(item)+1;
Path directory=output.resolve("cases").resolve(item.id());Files.createDirectories(directory);
System.out.println("Case "+attempted+"/"+cases.size()+": "+item.id());
boolean successful=true;
Expand All @@ -132,23 +143,27 @@ static void run(Path output, boolean smoke, List<String> libraries, int maxCases
if(code==0 && validRow(pending,item) && Files.isRegularFile(binary)) {
Storage.move(pending,json);
Storage.json(proof,Map.of("json_sha256",Storage.sha(json),"binary_sha256",Storage.sha(binary),"completed_utc",Instant.now().toString()));
saved=true;System.out.println(" Saved fork "+fork+" (full-precision checkpoint)");Storage.bundle(output);
saved=true;System.out.println(" Saved fork "+fork+" (full-precision checkpoint)");Storage.bundlePeriodically(output);
} else System.err.println(" Fork failed (exit "+code+"); attempt "+attempt+"/2; see case log.");
}
if(!saved) {successful=false;failed.add(Map.of("case",item.id(),"fork",fork,"reason","Two attempts failed; no timing result published for this case"));break;}
}
if(successful) {
JsonObject row=merge(directory);Storage.json(directory.resolve("result.json"),row);
JsonObject labelled=row.deepCopy();labelled.addProperty("requested_threads",item.threads);labelled.addProperty("profile",item.profile);labelled.addProperty("case_id",item.id());
combined.add(labelled);completed++;
JsonObject labelled=labelResult(row,item);
results.put(item.id(),labelled); combined=orderedResults(selected,results); completed++;
}
Storage.json(output.resolve("jmh.json"),combined);
Storage.jsonIfChanged(output.resolve("jmh.json"),combined);
Storage.json(output.resolve("failures.json"),failed);
Storage.json(output.resolve("progress.json"),Map.of("complete",completed==cases.size() && preflightFailures.isEmpty(),"completed_cases",completed,"attempted_cases",Math.min(attempted,cases.size()),"total_cases",cases.size(),"failed_cases",failed.size(),"updated_utc",Instant.now().toString()));
summary(output,combined);Storage.bundle(output);
summary(output,combined);Storage.bundlePeriodically(output);
}
} finally {Storage.bundle(output);}
if(completed==cases.size() && preflightFailures.isEmpty()) Storage.atomic(output.resolve("completed-utc.txt"),(Instant.now()+"\n").getBytes(java.nio.charset.StandardCharsets.UTF_8));
measurementsFinished=true;
} finally {if(!measurementsFinished) Storage.bundle(output);}
Storage.jsonIfChanged(output.resolve("jmh.json"),combined); summary(output,combined);
Storage.json(output.resolve("failures.json"),failed);
Storage.json(output.resolve("progress.json"),Map.of("complete",completed==cases.size() && preflightFailures.isEmpty(),"completed_cases",completed,"attempted_cases",selected.size(),"total_cases",cases.size(),"failed_cases",failed.size(),"updated_utc",Instant.now().toString()));
if(completed==cases.size() && preflightFailures.isEmpty() && !Files.exists(output.resolve("completed-utc.txt"))) Storage.atomic(output.resolve("completed-utc.txt"),(Instant.now()+"\n").getBytes(java.nio.charset.StandardCharsets.UTF_8));
Storage.json(output.resolve("suite-status.json"),Map.of("complete",completed==cases.size() && preflightFailures.isEmpty(),"completed_cases",completed,"total_cases",cases.size(),"failed_cases",failed.size(),"preflight_failures",preflightFailures,"suite",smoke?"SMOKE-NOT-FOR-PUBLICATION":"FULL"));
Storage.bundle(output);
System.out.println("Saved "+completed+"/"+cases.size()+" cases. Upload "+Storage.bundlePath(output));
Expand Down Expand Up @@ -197,7 +212,34 @@ static void summary(Path output,JsonArray rows) throws IOException {
StringBuilder csv=new StringBuilder("profile,requested_threads,library,operation,size_or_shape,mean_ms_per_op,jmh_99_9_error_ms,iteration_means\n");
for(JsonElement value:rows){JsonObject r=value.getAsJsonObject(),p=r.getAsJsonObject("params"),m=r.getAsJsonObject("primaryMetric");
csv.append(r.get("profile").getAsString()).append(',').append(r.get("requested_threads")).append(',').append(p.get("library").getAsString()).append(',').append(p.has("operation")?p.get("operation").getAsString():"multiplication").append(',').append(p.has("size")?p.get("size").getAsString():p.get("shape").getAsString()).append(',').append(m.get("score")).append(',').append(m.get("scoreError")).append(",6\n");}
Storage.atomic(output.resolve("summary.csv"),csv.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8));
Storage.atomicIfChanged(output.resolve("summary.csv"),csv.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8));
}
/** Verify every saved fork once. Only fully valid pairs may skip measurement. */
static Map<String,JsonObject> restoreCompleted(Path output,List<Case> cases) throws Exception {
Map<String,JsonObject> results=new LinkedHashMap<>();
for(Case item:cases) {
Path directory=output.resolve("cases").resolve(item.id());
if(validFork(directory.resolve("fork-1.json"),directory.resolve("fork-1.bin"),directory.resolve("fork-1-checkpoint.json"),item)
&& validFork(directory.resolve("fork-2.json"),directory.resolve("fork-2.bin"),directory.resolve("fork-2-checkpoint.json"),item))
results.put(item.id(),completedResult(directory,item));
}
return results;
}
static JsonObject completedResult(Path directory,Case item) throws Exception {
JsonObject row=merge(directory);
// A crash after the second fork may leave this derived export missing or stale.
Storage.jsonIfChanged(directory.resolve("result.json"),row);
return labelResult(row,item);
}
static JsonObject labelResult(JsonObject row,Case item) {
// Case result files are exports; do not rewrite them during a retained-case scan.
JsonObject labelled=row.deepCopy();labelled.addProperty("requested_threads",item.threads);labelled.addProperty("profile",item.profile);labelled.addProperty("case_id",item.id());
return labelled;
}
static JsonArray orderedResults(List<Case> cases,Map<String,JsonObject> results) {
JsonArray combined=new JsonArray();
for(Case item:cases)if(results.containsKey(item.id()))combined.add(results.get(item.id()));
return combined;
}
private BenchmarkMain() {}
}
Loading
Loading