From aff1431b1fd50a90cd6429cf4ad07fcb553229ae Mon Sep 17 00:00:00 2001 From: haksunli Date: Wed, 4 Nov 2026 12:16:58 -0500 Subject: [PATCH 1/3] Resume benchmark checkpoints in one scan and throttle ZIP exports --- .github/workflows/benchmark-application.yml | 38 +++++++++--- benchmark/LOCAL-RUN.md | 14 +++++ .../nm/benchmark/launcher/BenchmarkMain.java | 61 +++++++++++++++---- .../nm/benchmark/launcher/ResumeUpgrade.java | 59 ++++++++++++++++++ .../dev/nm/benchmark/launcher/Storage.java | 26 +++++++- .../dev/nm/benchmark/launcher/ResumeTest.java | 52 ++++++++++++++++ benchmark/pom.xml | 2 +- benchmark/run-benchmark.ps1 | 21 ++++++- 8 files changed, 251 insertions(+), 22 deletions(-) create mode 100644 benchmark/launcher/src/main/java/dev/nm/benchmark/launcher/ResumeUpgrade.java create mode 100644 benchmark/launcher/src/test/java/dev/nm/benchmark/launcher/ResumeTest.java diff --git a/.github/workflows/benchmark-application.yml b/.github/workflows/benchmark-application.yml index 66c690c..c215b57 100644 --- a/.github/workflows/benchmark-application.yml +++ b/.github/workflows/benchmark-application.yml @@ -39,18 +39,42 @@ jobs: - name: Build application and test checkpoint/ZIP safeguards run: mvn -B -f benchmark/pom.xml package - name: Verify automatic preparation and all library/profile adapters - run: java -Xmx2g -jar benchmark/target/nmdev-benchmark-1.3.0.jar --smoke --output smoke-results + run: java -Xmx2g -jar benchmark/target/nmdev-benchmark-1.3.1.jar --smoke --output smoke-results - name: Verify arbitrary local NMDEV filename and recorded binary origin shell: pwsh run: | Copy-Item smoke-results/.runtime/lib/nmdev-selected.jar 'selected NMDEV.jar' - java -Xmx2g -jar benchmark/target/nmdev-benchmark-1.3.0.jar --nmdev-jar 'selected NMDEV.jar' --smoke --libraries 'NMDEV,COMMONS,COMMONS_BLOCK' --max-cases 1 --output local-jar-results + java -Xmx2g -jar benchmark/target/nmdev-benchmark-1.3.1.jar --nmdev-jar 'selected NMDEV.jar' --smoke --libraries 'NMDEV,COMMONS,COMMONS_BLOCK' --max-cases 1 --output local-jar-results if ($LASTEXITCODE -ne 0) { throw 'Local-JAR smoke failed' } $expected = (Get-FileHash 'selected NMDEV.jar' -Algorithm SHA256).Hash.ToLowerInvariant() $recorded = Get-Content local-jar-results/binary-provenance.json -Raw | ConvertFrom-Json if ($recorded.nmdev_sha256 -ne $expected -or $recorded.nmdev_original_filename -ne 'selected NMDEV.jar') { throw 'Wrong NMDEV binary provenance' } - - name: Verify saved forks are reused - run: java -Xmx2g -jar benchmark/target/nmdev-benchmark-1.3.0.jar --smoke --output smoke-results + - name: Verify fast resume does not iterate completed cases + shell: pwsh + run: | + $resumed = java -Xmx2g -jar benchmark/target/nmdev-benchmark-1.3.1.jar --smoke --output smoke-results + $resumed | Write-Host + if ($LASTEXITCODE -ne 0) { throw 'Resume failed' } + if ($resumed -match '^Case [0-9]+/') { throw 'Resume revisited completed case loop' } + if (-not ($resumed -match '^Resume: verified [1-9]')) { throw 'No completed checkpoints restored' } + - name: Verify published 1.3.0 run upgrades without changing saved forks + shell: pwsh + run: | + Invoke-WebRequest -UseBasicParsing 'https://github.com/nmltd/nmdev-examples/releases/download/benchmark-v1.3.0/nmdev-benchmark-1.3.0.jar' -OutFile original-1.3.0.jar + if ((Get-FileHash original-1.3.0.jar -Algorithm SHA256).Hash.ToLowerInvariant() -ne 'a6b58052fedb17ea91f4c2812ac816708c700dfc386a9e3019306e9a26fde9cf') { throw 'Original binary hash mismatch' } + New-Item -ItemType Directory upgrade-results | Out-Null + Copy-Item local-jar-results/.runtime upgrade-results/.runtime -Recurse + java -Xmx2g -jar original-1.3.0.jar --nmdev-jar 'selected NMDEV.jar' --smoke --libraries 'NMDEV,COMMONS,COMMONS_BLOCK' --max-cases 1 --output upgrade-results + if ($LASTEXITCODE -ne 0) { throw 'Original run failed' } + $before = @{} + Get-ChildItem upgrade-results/cases -Recurse -File | Where-Object { $_.Name -like 'fork-*' -and $_.Extension -ne '.log' } | ForEach-Object { $before[$_.FullName] = (Get-FileHash $_.FullName -Algorithm SHA256).Hash } + java -cp benchmark/target/nmdev-benchmark-1.3.1.jar dev.nm.benchmark.launcher.ResumeUpgrade original-1.3.0.jar upgrade-results + if ($LASTEXITCODE -ne 0) { throw 'Upgrade verification failed' } + $resumed = java -Xmx2g -jar benchmark/target/nmdev-benchmark-1.3.1.jar --nmdev-jar 'selected NMDEV.jar' --smoke --libraries 'NMDEV,COMMONS,COMMONS_BLOCK' --max-cases 1 --output upgrade-results + $resumed | Write-Host + if ($LASTEXITCODE -ne 0 -or ($resumed -match '^Case [0-9]+/')) { throw 'Upgraded run repeated a completed case' } + foreach ($path in $before.Keys) { if ((Get-FileHash $path -Algorithm SHA256).Hash -ne $before[$path]) { throw 'Saved fork changed during upgrade/resume' } } + - uses: actions/upload-artifact@v4 if: always() with: @@ -61,7 +85,7 @@ jobs: if: matrix.os == 'ubuntu-24.04' && success() with: name: nmdev-benchmark-application - path: benchmark/target/nmdev-benchmark-1.3.0.jar + path: benchmark/target/nmdev-benchmark-1.3.1.jar release: if: github.event_name == 'push' && github.ref == 'refs/heads/main' @@ -80,5 +104,5 @@ jobs: GH_TOKEN: ${{ github.token }} run: | cd release - sha256sum nmdev-benchmark-1.3.0.jar > SHA256SUMS - gh release create benchmark-v1.3.0 nmdev-benchmark-1.3.0.jar SHA256SUMS ../benchmark/run-benchmark.ps1 --repo nmltd/nmdev-examples --target "$GITHUB_SHA" --title 'Reusable benchmark application 1.3.0' --notes 'Select any NMDEV binary with --nmdev-jar. Full 1,260-case suite across eleven libraries including both Commons matrix implementations. Released checksum-pinned Commons Math 4.0-beta1 dependencies. Windows and Linux application smoke, local-JAR selection and resume checks passed for this exact application build. Full benchmark timings are run locally.' + sha256sum nmdev-benchmark-1.3.1.jar > SHA256SUMS + gh release create benchmark-v1.3.1 nmdev-benchmark-1.3.1.jar SHA256SUMS ../benchmark/run-benchmark.ps1 --repo nmltd/nmdev-examples --target "$GITHUB_SHA" --title 'Faster benchmark resume 1.3.1' --notes 'Select any NMDEV binary with --nmdev-jar. Full 1,260-case suite across eleven libraries including both Commons matrix implementations. Released checksum-pinned Commons Math 4.0-beta1 dependencies. Windows and Linux application smoke, local-JAR selection and resume checks passed for this exact application build. Fast checkpoint scan and periodic ZIP exports. Published 1.3.0 resume migration verified with unchanged saved fork hashes. Full benchmark timings are run locally.' diff --git a/benchmark/LOCAL-RUN.md b/benchmark/LOCAL-RUN.md index 72134a1..d32fdb0 100644 --- a/benchmark/LOCAL-RUN.md +++ b/benchmark/LOCAL-RUN.md @@ -1,3 +1,17 @@ +# Faster resume and exports (application 1.3.1) + +Completed cases are verified in one scan and collected without re-entering the measurement loop. Reports are rebuilt once before unfinished work; fully completed resumes only export at the end. Fork checkpoints are still written and synchronized immediately. During measurement, full ZIP snapshots are limited to once a minute; orderly completion and interruption force a final snapshot. Brief Windows AccessDenied failures during file replacement are retried without deleting the previous file or ignoring checkpoint errors. + +The checked-in PowerShell entry point downloads application 1.3.1. Existing 1.3.0 result directories continue using their original runner unless an explicit verified resume upgrade is requested. To upgrade a stopped published 1.3.0 run: + +```powershell +powershell -ExecutionPolicy Bypass -File run-benchmark.ps1 -NmdevVersion 1.2.10 -UpgradeResume +``` + +Stop an active runner with Ctrl+C first. The helper verifies the original published 1.3.0 SHA-256 and byte-compares the application JARs: numerical harness sources, dependencies, case-model class and other application classes must match. Only the audited resume/export classes, added upgrade helper and application version metadata may differ. It locks the result directory, backs up request.json, records provenance and updates only the application hash. Saved fork files, measurements, selected NMDEV binary and case specifications are not changed. Full normal machine/configuration/dependency and fork integrity checks run after migration. An unknown original build or any unexpected JAR entry difference is rejected. + +--- + # Reusable benchmark application 1.3.0 From PowerShell, run `benchmark/run-benchmark.ps1`. It downloads the checksum-verified prebuilt runner and NMDEV 1.2.10, then starts the complete suite. A full JDK 17+ is required; Maven and source edits are unnecessary. diff --git a/benchmark/launcher/src/main/java/dev/nm/benchmark/launcher/BenchmarkMain.java b/benchmark/launcher/src/main/java/dev/nm/benchmark/launcher/BenchmarkMain.java index bcc72fb..b84d6ce 100644 --- a/benchmark/launcher/src/main/java/dev/nm/benchmark/launcher/BenchmarkMain.java +++ b/benchmark/launcher/src/main/java/dev/nm/benchmark/launcher/BenchmarkMain.java @@ -54,7 +54,7 @@ public static void main(String[] args) throws Exception { case "--smoke": smoke=true; break; case "--max-cases": maxCases=Integer.parseInt(args[++i]); break; case "--libraries": libraries=List.of(args[++i].split(",")); if(!LIBRARIES.containsAll(libraries)) throw new IllegalArgumentException("Unknown library"); break; - case "--help": System.out.println("java -jar nmdev-benchmark-1.3.0.jar [--nmdev-jar path/to/nmdev.jar] [--output directory]\nNo arguments runs all 11 libraries (12 implementation variants) and all profiles. Repeat the same command to resume.\nDiagnostics only: --smoke [--libraries NMDEV,EJML] [--max-cases N]"); return; + case "--help": System.out.println("java -jar nmdev-benchmark-1.3.1.jar [--nmdev-jar path/to/nmdev.jar] [--output directory]\nNo arguments runs all 11 libraries (12 implementation variants) and all profiles. Repeat the same command to resume.\nDiagnostics only: --smoke [--libraries NMDEV,EJML] [--max-cases N]"); return; default: throw new IllegalArgumentException("Unknown argument: "+args[i]); } if (!smoke && !libraries.equals(LIBRARIES)) throw new IllegalArgumentException("--libraries is a diagnostic option requiring --smoke"); @@ -99,7 +99,7 @@ static void run(Path output, boolean smoke, List libraries, int maxCases request.put("child_jvm_options","-Xms256m -Xmx1g; ActiveProcessorCount and benchmark.threads per case; F2J BLAS/LAPACK/ARPACK forced"); Path manifest=output.resolve("request.json"); JsonElement specification=Storage.JSON.toJsonTree(request); if(Files.exists(manifest) && !Storage.read(manifest).equals(specification)) throw new IOException("Run configuration, binaries or machine changed. Use a different --output directory; existing checkpoints were preserved."); - Storage.json(manifest,specification); Files.deleteIfExists(output.resolve("completed-utc.txt")); + Storage.json(manifest,specification); Storage.atomic(output.resolve("README.md"),("# NM Dev benchmark output\n\nSuite: "+request.get("suite")+". Profiles ending in -all-cores request all JVM-visible logical processors recorded in request.json. Single-threaded APIs remain single threaded; requested workers do not establish actual utilization. COMMONS uses Array2DRowRealMatrix; COMMONS_BLOCK uses BlockRealMatrix, constructed before timing. Compare matching operations and dimensions across profiles; do not pool thread settings. All measurements are local to the machine recorded in machine.json.\n\nCase checkpoints include two independently warmed JVM forks. Fork JSON is preserved verbatim; combined JSON uses JMH's own aggregators. Requested native thread counts are not independently audited.\n\nUpload the sibling *-results-.zip. It excludes licenses, downloaded binaries, credentials and private runtime cache. Partial runs are explicitly marked in progress.json. To resume, keep the complete local output directory, including .runtime and binary fork checkpoints, and run the same JAR again.\n").getBytes(java.nio.charset.StandardCharsets.UTF_8)); // Preflight failures are recorded; each timed fork independently validates its // own full-size inputs. An unsupported native backend does not hide other libraries. @@ -113,10 +113,21 @@ static void run(Path output, boolean smoke, List libraries, int maxCases } if(Storage.read(done).getAsJsonObject().get("exit_code").getAsInt()!=0) preflightFailures.add(name); } - JsonArray combined=new JsonArray(); List> failed=new ArrayList<>(); int completed=0,attempted=0; + List selected=cases.subList(0,Math.min(Math.max(maxCases,0),cases.size())); + Map results=restoreCompleted(output,selected); + List pendingCases=selected.stream().filter(item->!results.containsKey(item.id())).toList(); + JsonArray combined=orderedResults(selected,results); + List> failed=new ArrayList<>(); int completed=results.size(),attempted=0; + System.out.println("Resume: verified "+completed+" completed cases; "+pendingCases.size()+" cases need work in this invocation."); + if(!pendingCases.isEmpty() || !preflightFailures.isEmpty()) Files.deleteIfExists(output.resolve("completed-utc.txt")); + // Rebuild saved reports once, not once for each retained case. + if(completed>0 && !pendingCases.isEmpty()) { + Storage.json(output.resolve("jmh.json"),combined); summary(output,combined); + } + boolean measurementsFinished=false; try { - for(Case item:cases) { - if(attempted++>=maxCases) break; + for(Case item:pendingCases) { + attempted=cases.indexOf(item)+1; Path directory=output.resolve("cases").resolve(item.id());Files.createDirectories(directory); System.out.println("Case "+attempted+"/"+cases.size()+": "+item.id()); boolean successful=true; @@ -132,23 +143,27 @@ static void run(Path output, boolean smoke, List libraries, int maxCases if(code==0 && validRow(pending,item) && Files.isRegularFile(binary)) { Storage.move(pending,json); Storage.json(proof,Map.of("json_sha256",Storage.sha(json),"binary_sha256",Storage.sha(binary),"completed_utc",Instant.now().toString())); - saved=true;System.out.println(" Saved fork "+fork+" (full-precision checkpoint)");Storage.bundle(output); + saved=true;System.out.println(" Saved fork "+fork+" (full-precision checkpoint)");Storage.bundlePeriodically(output); } else System.err.println(" Fork failed (exit "+code+"); attempt "+attempt+"/2; see case log."); } if(!saved) {successful=false;failed.add(Map.of("case",item.id(),"fork",fork,"reason","Two attempts failed; no timing result published for this case"));break;} } if(successful) { JsonObject row=merge(directory);Storage.json(directory.resolve("result.json"),row); - JsonObject labelled=row.deepCopy();labelled.addProperty("requested_threads",item.threads);labelled.addProperty("profile",item.profile);labelled.addProperty("case_id",item.id()); - combined.add(labelled);completed++; + JsonObject labelled=labelResult(row,item); + results.put(item.id(),labelled); combined=orderedResults(selected,results); completed++; } Storage.json(output.resolve("jmh.json"),combined); Storage.json(output.resolve("failures.json"),failed); Storage.json(output.resolve("progress.json"),Map.of("complete",completed==cases.size() && preflightFailures.isEmpty(),"completed_cases",completed,"attempted_cases",Math.min(attempted,cases.size()),"total_cases",cases.size(),"failed_cases",failed.size(),"updated_utc",Instant.now().toString())); - summary(output,combined);Storage.bundle(output); + summary(output,combined);Storage.bundlePeriodically(output); } - } finally {Storage.bundle(output);} - if(completed==cases.size() && preflightFailures.isEmpty()) Storage.atomic(output.resolve("completed-utc.txt"),(Instant.now()+"\n").getBytes(java.nio.charset.StandardCharsets.UTF_8)); + measurementsFinished=true; + } finally {if(!measurementsFinished) Storage.bundle(output);} + Storage.json(output.resolve("jmh.json"),combined); summary(output,combined); + Storage.json(output.resolve("failures.json"),failed); + Storage.json(output.resolve("progress.json"),Map.of("complete",completed==cases.size() && preflightFailures.isEmpty(),"completed_cases",completed,"attempted_cases",selected.size(),"total_cases",cases.size(),"failed_cases",failed.size(),"updated_utc",Instant.now().toString())); + if(completed==cases.size() && preflightFailures.isEmpty() && !Files.exists(output.resolve("completed-utc.txt"))) Storage.atomic(output.resolve("completed-utc.txt"),(Instant.now()+"\n").getBytes(java.nio.charset.StandardCharsets.UTF_8)); Storage.json(output.resolve("suite-status.json"),Map.of("complete",completed==cases.size() && preflightFailures.isEmpty(),"completed_cases",completed,"total_cases",cases.size(),"failed_cases",failed.size(),"preflight_failures",preflightFailures,"suite",smoke?"SMOKE-NOT-FOR-PUBLICATION":"FULL")); Storage.bundle(output); System.out.println("Saved "+completed+"/"+cases.size()+" cases. Upload "+Storage.bundlePath(output)); @@ -199,5 +214,29 @@ static void summary(Path output,JsonArray rows) throws IOException { csv.append(r.get("profile").getAsString()).append(',').append(r.get("requested_threads")).append(',').append(p.get("library").getAsString()).append(',').append(p.has("operation")?p.get("operation").getAsString():"multiplication").append(',').append(p.has("size")?p.get("size").getAsString():p.get("shape").getAsString()).append(',').append(m.get("score")).append(',').append(m.get("scoreError")).append(",6\n");} Storage.atomic(output.resolve("summary.csv"),csv.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8)); } + /** Verify every saved fork once. Only fully valid pairs may skip measurement. */ + static Map restoreCompleted(Path output,List cases) throws Exception { + Map results=new LinkedHashMap<>(); + for(Case item:cases) { + Path directory=output.resolve("cases").resolve(item.id()); + if(validFork(directory.resolve("fork-1.json"),directory.resolve("fork-1.bin"),directory.resolve("fork-1-checkpoint.json"),item) + && validFork(directory.resolve("fork-2.json"),directory.resolve("fork-2.bin"),directory.resolve("fork-2-checkpoint.json"),item)) + results.put(item.id(),completedResult(directory,item)); + } + return results; + } + static JsonObject completedResult(Path directory,Case item) throws Exception { + return labelResult(merge(directory),item); + } + static JsonObject labelResult(JsonObject row,Case item) { + // Case result files are exports; do not rewrite them during a retained-case scan. + JsonObject labelled=row.deepCopy();labelled.addProperty("requested_threads",item.threads);labelled.addProperty("profile",item.profile);labelled.addProperty("case_id",item.id()); + return labelled; + } + static JsonArray orderedResults(List cases,Map results) { + JsonArray combined=new JsonArray(); + for(Case item:cases)if(results.containsKey(item.id()))combined.add(results.get(item.id())); + return combined; + } private BenchmarkMain() {} } diff --git a/benchmark/launcher/src/main/java/dev/nm/benchmark/launcher/ResumeUpgrade.java b/benchmark/launcher/src/main/java/dev/nm/benchmark/launcher/ResumeUpgrade.java new file mode 100644 index 0000000..042c670 --- /dev/null +++ b/benchmark/launcher/src/main/java/dev/nm/benchmark/launcher/ResumeUpgrade.java @@ -0,0 +1,59 @@ +package dev.nm.benchmark.launcher; + +import com.google.gson.JsonObject; +import java.io.IOException; +import java.nio.channels.*; +import java.nio.file.*; +import java.time.Instant; +import java.util.*; +import java.util.jar.*; + +/** Audited 1.3.0 -> 1.3.1 export/resume upgrade. Never edits a saved fork. */ +public final class ResumeUpgrade { + static final String ORIGINAL_SHA="a6b58052fedb17ea91f4c2812ac816708c700dfc386a9e3019306e9a26fde9cf"; + public static void main(String[] args) throws Exception { + if(args.length!=2)throw new IllegalArgumentException("Usage: ResumeUpgrade original-1.3.0.jar output-directory"); + Path oldJar=Path.of(args[0]).toAbsolutePath(),output=Path.of(args[1]).toAbsolutePath(),newJar=Dependencies.application(); + if(!ORIGINAL_SHA.equals(Storage.sha(oldJar)))throw new IOException("Original application is not the published 1.3.0 binary"); + try(var channel=FileChannel.open(output.resolve(".runner.lock"),StandardOpenOption.CREATE,StandardOpenOption.WRITE); + FileLock lock=channel.tryLock()) { + if(lock==null)throw new IOException("Stop the active benchmark before upgrading resume/export code"); + Path manifest=output.resolve("request.json");JsonObject request=Storage.read(manifest).getAsJsonObject(); + String newHash=Storage.sha(newJar),oldHash=request.get("application_sha256").getAsString(); + if(newHash.equals(oldHash)) {System.out.println("This run already uses the faster resume implementation.");return;} + if(!ORIGINAL_SHA.equals(oldHash))throw new IOException("Saved run did not use the published 1.3.0 application"); + verify(oldJar,newJar); + Path backup=output.resolve("request-before-resume-"+oldHash+".json"); + if(!Files.exists(backup))Storage.atomic(backup,Files.readAllBytes(manifest)); + Storage.json(output.resolve("resume-upgrade-"+newHash+".json"),Map.of("utc",Instant.now().toString(), + "old_application_sha256",oldHash,"new_application_sha256",newHash, + "verification","Numerical harness, dependencies, case model and other application classes byte-identical; audited resume/export classes and application version metadata only; saved forks untouched")); + request.addProperty("application_sha256",newHash);Storage.json(manifest,request); + System.out.println("Verified resume/export upgrade. Existing fork checkpoints were preserved."); + } + } + static void verify(Path oldJar,Path newJar) throws IOException { + String prefix="dev/nm/benchmark/launcher/"; + Set allowed=Set.of(prefix+"BenchmarkMain.class",prefix+"Storage.class",prefix+"ResumeUpgrade.class"); + Set versioned=Set.of("META-INF/maven/dev.nm/nmdev-benchmark/pom.xml","META-INF/maven/dev.nm/nmdev-benchmark/pom.properties"); + try(JarFile oldFile=new JarFile(oldJar.toFile());JarFile newFile=new JarFile(newJar.toFile())) { + Map oldEntries=entries(oldFile),newEntries=entries(newFile); + Set names=new TreeSet<>(oldEntries.keySet());names.addAll(newEntries.keySet()); + for(String name:names) { + if(allowed.contains(name))continue; + JarEntry before=oldEntries.get(name),after=newEntries.get(name); + if(before==null || after==null)throw new IOException("Unexpected JAR entry difference: "+name); + try(var a=oldFile.getInputStream(before);var b=newFile.getInputStream(after)) { + byte[] previous=a.readAllBytes(),next=b.readAllBytes(); + if(Arrays.equals(previous,next))continue; + if(versioned.contains(name) && new String(previous,java.nio.charset.StandardCharsets.UTF_8).replace("1.3.0","1.3.1").equals(new String(next,java.nio.charset.StandardCharsets.UTF_8)))continue; + throw new IOException("Measurement/dependency content changed: "+name); + } + } + } + } + private static Map entries(JarFile jar) { + Map result=new HashMap<>();jar.stream().filter(e->!e.isDirectory()).forEach(e->result.put(e.getName(),e));return result; + } + private ResumeUpgrade() {} +} diff --git a/benchmark/launcher/src/main/java/dev/nm/benchmark/launcher/Storage.java b/benchmark/launcher/src/main/java/dev/nm/benchmark/launcher/Storage.java index 5036d57..7c3b261 100644 --- a/benchmark/launcher/src/main/java/dev/nm/benchmark/launcher/Storage.java +++ b/benchmark/launcher/src/main/java/dev/nm/benchmark/launcher/Storage.java @@ -20,8 +20,23 @@ static void atomic(Path path, byte[] bytes) throws IOException { move(temp, path); } static void move(Path from, Path to) throws IOException { - try { Files.move(from, to, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } - catch (AtomicMoveNotSupportedException ex) { Files.move(from, to, StandardCopyOption.REPLACE_EXISTING); } + // Windows scanners/readers can briefly deny replacement of a closed file. + // Retry the rename; never delete the previous destination or weaken checkpoints. + for(int attempt=0;;attempt++) { + try { + try {Files.move(from,to,StandardCopyOption.ATOMIC_MOVE,StandardCopyOption.REPLACE_EXISTING);} + catch(AtomicMoveNotSupportedException ex) {Files.move(from,to,StandardCopyOption.REPLACE_EXISTING);} + return; + } catch(AccessDeniedException ex) { + if(attempt==7)throw ex; + try {Thread.sleep(100L*(attempt+1));} + catch(InterruptedException interrupted) { + Thread.currentThread().interrupt(); + java.io.InterruptedIOException failure=new java.io.InterruptedIOException("Interrupted while replacing "+to); + failure.initCause(ex);throw failure; + } + } + } } static String sha(Path path) throws IOException { try { @@ -57,11 +72,18 @@ static synchronized Path bundlePath(Path output) throws IOException { throw new IOException("Invalid ZIP timestamp in "+stamp); return output.resolveSibling(output.getFileName()+"-results-"+timestamp+".zip"); } + private static final Map LAST_BUNDLE=new HashMap<>(); + /** Durable fork checkpoints are immediate; full ZIP exports are at most once a minute. */ + static synchronized void bundlePeriodically(Path output) throws IOException { + long now=System.nanoTime();Long previous=LAST_BUNDLE.get(output.toAbsolutePath().normalize()); + if(previous==null || now-previous>=java.util.concurrent.TimeUnit.SECONDS.toNanos(60))bundle(output); + } static synchronized void bundle(Path output) throws IOException { bundle(output, null); } // Snapshot callback enables deterministic tests of disappearing source files. static synchronized void bundle(Path output, Runnable afterSnapshot) throws IOException { + LAST_BUNDLE.put(output.toAbsolutePath().normalize(),System.nanoTime()); try { bundleSnapshot(output, afterSnapshot); } catch (IOException | java.io.UncheckedIOException | SecurityException ex) { System.err.println("WARNING: Results ZIP refresh failed: " + ex diff --git a/benchmark/launcher/src/test/java/dev/nm/benchmark/launcher/ResumeTest.java b/benchmark/launcher/src/test/java/dev/nm/benchmark/launcher/ResumeTest.java new file mode 100644 index 0000000..30de1e3 --- /dev/null +++ b/benchmark/launcher/src/test/java/dev/nm/benchmark/launcher/ResumeTest.java @@ -0,0 +1,52 @@ +package dev.nm.benchmark.launcher; + +import java.nio.file.*; +import java.util.*; +import java.util.jar.*; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import static org.junit.jupiter.api.Assertions.*; + +class ResumeTest { + @TempDir Path temporary; + @Test void periodicArchivesDoNotRecompressSavedCheckpointsButFinalSnapshotIsImmediate() throws Exception { + Path output=temporary.resolve("run");Files.createDirectories(output); + Storage.json(output.resolve("checkpoint.json"),Map.of("saved",1)); + Storage.bundlePeriodically(output); + Path zip=Storage.bundlePath(output);byte[] initial=Files.readAllBytes(zip); + Storage.json(output.resolve("checkpoint.json"),Map.of("saved",2)); + for(int i=0;i<640;i++)Storage.bundlePeriodically(output); + assertArrayEquals(initial,Files.readAllBytes(zip)); + assertTrue(Files.readString(output.resolve("checkpoint.json")).contains("2")); + Storage.bundle(output); + assertFalse(Arrays.equals(initial,Files.readAllBytes(zip))); + } + @Test void upgradeRejectsChangedNumericalSourcesAndDependencyBytes() throws Exception { + Path old=jar("old.jar",Map.of("harness-src/MatrixBenchmark.java","original","dep.class","same","dev/nm/benchmark/launcher/BenchmarkMain.class","old")); + Path good=jar("good.jar",Map.of("harness-src/MatrixBenchmark.java","original","dep.class","same","dev/nm/benchmark/launcher/BenchmarkMain.class","new","dev/nm/benchmark/launcher/ResumeUpgrade.class","helper")); + Path sourceChanged=jar("source.jar",Map.of("harness-src/MatrixBenchmark.java","changed","dep.class","same")); + Path dependencyChanged=jar("dep.jar",Map.of("harness-src/MatrixBenchmark.java","original","dep.class","different")); + assertDoesNotThrow(()->ResumeUpgrade.verify(old,good)); + assertThrows(java.io.IOException.class,()->ResumeUpgrade.verify(old,sourceChanged)); + assertThrows(java.io.IOException.class,()->ResumeUpgrade.verify(old,dependencyChanged)); + } + @Test void resultOrderingIsStableWhenAnEarlierCaseIsRecoveredAfterLaterCompletedCases() { + var early=new BenchmarkMain.Case("baseline",1,"NMDEV","scale","64"); + var later=new BenchmarkMain.Case("baseline",1,"EJML","scale","64"); + var a=new com.google.gson.JsonObject();a.addProperty("case_id",early.id()); + var b=new com.google.gson.JsonObject();b.addProperty("case_id",later.id()); + Map results=new LinkedHashMap<>();results.put(later.id(),b);results.put(early.id(),a); + var ordered=BenchmarkMain.orderedResults(List.of(early,later),results); + assertEquals(early.id(),ordered.get(0).getAsJsonObject().get("case_id").getAsString()); + assertEquals(later.id(),ordered.get(1).getAsJsonObject().get("case_id").getAsString()); + } + private Path jar(String name,Map entries) throws Exception { + Path path=temporary.resolve(name); + try(var jar=new JarOutputStream(Files.newOutputStream(path))) { + for(var entry:entries.entrySet()) { + jar.putNextEntry(new JarEntry(entry.getKey()));jar.write(entry.getValue().getBytes(java.nio.charset.StandardCharsets.UTF_8));jar.closeEntry(); + } + } + return path; + } +} diff --git a/benchmark/pom.xml b/benchmark/pom.xml index f16abff..909569b 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -1,6 +1,6 @@ 4.0.0 - dev.nmnmdev-benchmark1.3.0 + dev.nmnmdev-benchmark1.3.1 NM Dev unattended benchmark application 17UTF-8