From 65e67c1e171e2e0fb470b95c20b873897defa63c Mon Sep 17 00:00:00 2001 From: Samyak Jhaveri Date: Mon, 24 Aug 2026 12:45:51 -0700 Subject: [PATCH] ParBench public snapshot 2026-08-24 --- .github/ISSUE_TEMPLATE/bug-report.yml | 77 +++++++ .github/ISSUE_TEMPLATE/new-suite.yml | 91 ++++++++ .github/PAGES_SETUP.md | 25 --- .github/PULL_REQUEST_TEMPLATE.md | 21 ++ .github/workflows/ci.yml | 50 +++++ .github/workflows/deploy-pages.yml | 53 ----- CITATION.cff | 76 +++++++ CONTRIBUTING.md | 67 ++++++ README.md | 89 +++++--- artifact/README.md | 89 ++++++-- artifact/croissant.json | 75 ++++++- docs/benchmark-tree-pins.md | 127 +++++++++--- docs/running-evals.md | 2 +- pyproject.toml | 2 + requirements.txt | 1 + scripts/check_generated_registry.py | 206 ------------------- scripts/pre-commit.sh | 12 -- scripts/run_public_tests.sh | 51 +++++ tests/test_generated_registry_enforcement.py | 123 ----------- 19 files changed, 731 insertions(+), 506 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug-report.yml create mode 100644 .github/ISSUE_TEMPLATE/new-suite.yml delete mode 100644 .github/PAGES_SETUP.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/deploy-pages.yml create mode 100644 CITATION.cff create mode 100644 CONTRIBUTING.md delete mode 100644 scripts/check_generated_registry.py delete mode 100755 scripts/pre-commit.sh create mode 100755 scripts/run_public_tests.sh delete mode 100644 tests/test_generated_registry_enforcement.py diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 00000000..c4a8788a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,77 @@ +name: Bug report +description: Report a spec, build, run, or verification failure +title: "[bug] " +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Build and verify failures are usually toolchain-specific, so the compiler and OS + versions matter as much as the error text. Please fill in every field. + + Some specs are known to fail and are excluded from evaluation batches by code + (`EXCLUDED_SPECS` in `harness/constants.py`). Please check that list first. + + - type: input + id: spec + attributes: + label: Spec + description: The spec's unique_id, or the spec file path. + placeholder: rodinia-nw-omp + validations: + required: true + + - type: input + id: platform + attributes: + label: Platform and OS + description: CPU or GPU, distribution, and kernel or OS version. + placeholder: Ubuntu 24.04, x86_64, NVIDIA RTX 4070 + validations: + required: true + + - type: input + id: compiler + attributes: + label: Compiler and toolchain versions + description: Output of `g++ --version`, `nvcc --version`, or the OpenCL runtime version. + placeholder: g++ 13.2.0, nvcc 12.4 + validations: + required: true + + - type: textarea + id: command + attributes: + label: Exact command + description: The command you ran, verbatim, including the working directory. + render: shell + placeholder: | + cd ParBench + python3 -m harness verify specs/rodinia-nw-omp.json + validations: + required: true + + - type: textarea + id: output + attributes: + label: Full output + description: The complete output, not an excerpt. Attach a file if it is very long. + render: shell + validations: + required: true + + - type: textarea + id: expected + attributes: + label: Expected versus actual + description: What you expected to happen, and what happened instead. + validations: + required: true + + - type: input + id: commit + attributes: + label: ParBench commit + description: Output of `git rev-parse --short HEAD`. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/new-suite.yml b/.github/ISSUE_TEMPLATE/new-suite.yml new file mode 100644 index 00000000..943df070 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/new-suite.yml @@ -0,0 +1,91 @@ +name: New benchmark suite +description: Propose adding a benchmark suite to ParBench +title: "[suite] " +labels: ["new-suite"] +body: + - type: markdown + attributes: + value: | + Please read [docs/adding-a-suite.md](https://github.com/Scientific-Computing-Lab/ParBench/blob/main/docs/adding-a-suite.md) + and [CONTRIBUTING.md](https://github.com/Scientific-Computing-Lab/ParBench/blob/main/CONTRIBUTING.md) + before filing. Open this issue before + writing specs, so the pinning and licensing questions are settled first. + + - type: input + id: suite + attributes: + label: Suite name + description: The slug that will prefix every unique_id, lowercase. + placeholder: mysuite + validations: + required: true + + - type: input + id: upstream + attributes: + label: Upstream repository URL + validations: + required: true + + - type: input + id: license + attributes: + label: Upstream license + description: ParBench does not vendor benchmark source, so the suite keeps its own license. + placeholder: BSD-3-Clause + validations: + required: true + + - type: input + id: commit + attributes: + label: Pinned commit SHA + description: The full upstream commit the specs will be written against. + validations: + required: true + + - type: checkboxes + id: apis + attributes: + label: APIs covered + options: + - label: CUDA + - label: OpenMP + - label: OpenCL + - label: OpenMP target offload + validations: + required: true + + - type: textarea + id: kernels + attributes: + label: Proposed kernels + description: One line per kernel, with the API variants you intend to write specs for. + placeholder: | + stencil - CUDA, OpenMP + spmv - CUDA, OpenMP, OpenCL + validations: + required: true + + - type: textarea + id: oracle + attributes: + label: Oracle plan + description: >- + For each kernel, the correctness oracle you intend to declare and its strength: + strong (file_hash or file_diff on a computed output), medium (numeric_comparison + against a tolerance), or weak (stdout_pattern plus exit_code). If a kernel can only + carry a weak oracle, say why - for example, the upstream references disagree across + APIs. + validations: + required: true + + - type: textarea + id: build + attributes: + label: Build and run requirements + description: >- + Compilers, external data downloads, GPU requirements, and expected runtime per + kernel. Note anything that cannot run on a CPU-only machine. + validations: + required: false diff --git a/.github/PAGES_SETUP.md b/.github/PAGES_SETUP.md deleted file mode 100644 index 0faab0cd..00000000 --- a/.github/PAGES_SETUP.md +++ /dev/null @@ -1,25 +0,0 @@ -# GitHub Pages Setup - -## Setting the Pages Password - -The visualizations site is password-protected via staticrypt. Before the workflow can deploy, set the password secret: - -### Via GitHub UI -1. Go to repo Settings → Secrets and variables → Actions -2. Click "New repository secret" -3. Name: `PAGES_PASSWORD` -4. Value: your chosen password -5. Click "Add secret" - -### Via GitHub API (if gh CLI is available) -```bash -gh secret set PAGES_PASSWORD -``` - -## Re-deploying -After setting the secret, trigger a new deployment: -- Push any change to `visualizations/` -- Or go to Actions → "Deploy Visualizations to GitHub Pages" → "Run workflow" - -## Changing the Password -Update the `PAGES_PASSWORD` secret and re-run the workflow. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..be0241a3 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,21 @@ +## What this changes + + + +## Checklist + +- [ ] `bash scripts/run_public_tests.sh` passes. +- [ ] Every spec I added or changed validates: + `python3 scripts/validate_schema.py --spec specs/.json` exits 0. +- [ ] Every spec I added or changed verifies against its pristine baseline: + `python3 -m harness verify specs/.json`. +- [ ] `manifest.jsonl` changes are appended lines only. No existing entry was edited or + removed. +- [ ] Run arguments were read from the source's argument parser, not from documentation + (`python3 scripts/spec_tools/check_spec_argc.py --all`). +- [ ] No machine-specific paths, home directories, hostnames, credentials, or local + `config/paths.json` values appear in the diff. + +## Platform tested + + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..2bc4ac22 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,50 @@ +name: CI + +# CPU-only sanity gate for the public repo: the spec schema validates, the +# harness package imports, and the unit suite passes on a clean clone with no +# benchmark trees fetched and no GPU. It deliberately does NOT build or run a +# kernel - every suite tree (rodinia, HeCBench, mixbench, xsbench, rsbench) is +# fetched separately, see docs/benchmark-tree-pins.md. + +on: + pull_request: + push: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + # The rodinia submodule (~101 MB) buys exactly one extra passing test, + # and only after a local layout fixup, so CI skips it. Measured on a + # clean clone of the public repo: 589 passed either way. + submodules: false + + - uses: actions/setup-python@v5 + with: + # pyproject requires >=3.12; requirements-lock.txt was pinned on + # Ubuntu 24.04 / Python 3.12.3. + python-version: '3.12' + cache: pip + cache-dependency-path: requirements-lock.txt + + - name: Install + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -r requirements-lock.txt + python3 -m pip install -e . + + - name: Validate a spec against the schema + run: python3 scripts/validate_schema.py --spec specs/rodinia-nw-omp.json + + - name: Import the harness package + run: python3 -c "import harness" + + - name: Unit tests + # The exclusion list lives in scripts/run_public_tests.sh, the same + # runner CONTRIBUTING.md and the PR template tell contributors to use. + # Measured on a clean public clone: without the exclusions the suite is + # 43 failed / 589 passed / 112 skipped; with them it exits 0. + run: bash scripts/run_public_tests.sh diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml deleted file mode 100644 index d3979a07..00000000 --- a/.github/workflows/deploy-pages.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Deploy Visualizations to GitHub Pages - -on: - push: - branches: [main] - paths: ['visualizations/**'] - workflow_dispatch: - -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: pages - cancel-in-progress: false - -jobs: - deploy: - runs-on: ubuntu-latest - timeout-minutes: 10 - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - steps: - - uses: actions/checkout@v4 - - - name: Install staticrypt - run: npm install -g staticrypt - - - name: Encrypt HTML files with password - # staticrypt v3: encrypts HTML files with AES-256, adds a password prompt page - # -d encrypted/ writes encrypted files to encrypted/ subdirectory (staticrypt default) - # --remember 1 stores auth in localStorage for 1 day across page navigations - # Then we move encrypted files back over the originals for deployment - env: - PAGES_PASSWORD: ${{ secrets.PAGES_PASSWORD }} - run: | - if [ -z "$PAGES_PASSWORD" ]; then - echo "::error::PAGES_PASSWORD secret is not set. Go to Settings → Secrets → Actions and add PAGES_PASSWORD before deploying." - exit 1 - fi - cd visualizations - staticrypt *.html --password "$PAGES_PASSWORD" --remember 1 --short - mv encrypted/*.html . - rmdir encrypted - - - uses: actions/configure-pages@v5 - - uses: actions/upload-pages-artifact@v4 - with: - path: visualizations - - id: deployment - uses: actions/deploy-pages@v4 diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 00000000..41a3b177 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,76 @@ +cff-version: 1.2.0 +message: "If you use ParBench, please cite both the software and the paper below." +title: "ParBench" +abstract: >- + An evaluation harness for LLM-based parallel code translation. ParBench + describes each benchmark kernel in a machine-readable spec and then builds, + runs, and verifies every LLM-produced translation, so that results are + comparable across models, benchmark suites, and translation directions + (CUDA, OpenMP, OpenCL, and OpenMP target offload). +type: software +version: "0.1.0" +date-released: "2026-08-24" +license: MIT +repository-code: "https://github.com/Scientific-Computing-Lab/ParBench" +url: "https://github.com/Scientific-Computing-Lab/ParBench" +keywords: + - benchmark + - llm-evaluation + - parallel-computing + - code-translation + - cuda + - openmp + - opencl +authors: + - given-names: "Samyak" + family-names: "Jhaveri" + affiliation: "University of California, Irvine" + - given-names: "Erel" + family-names: "Kaplan" + affiliation: "Technion" + - given-names: "Tom" + family-names: "Yotam" + affiliation: "Code Metal" + - given-names: "Le" + family-names: "Chen" + affiliation: "Argonne National Laboratory" + - given-names: "Tomer" + family-names: "Bitan" + affiliation: "Technion" + - given-names: "Niranjan" + family-names: "Hasabnis" + affiliation: "Code Metal" + - given-names: "Gal" + family-names: "Oren" + affiliation: "Stanford University, Technion" +preferred-citation: + type: conference-paper + title: >- + ParBench: A Benchmark for Reliable Evaluation of LLM Parallel Code + Translation + year: 2026 + collection-title: "Proceedings of the Conference on Neural Information Processing Systems (NeurIPS 2026)" + conference: + name: "Conference on Neural Information Processing Systems (NeurIPS 2026)" + authors: + - given-names: "Samyak" + family-names: "Jhaveri" + affiliation: "University of California, Irvine" + - given-names: "Erel" + family-names: "Kaplan" + affiliation: "Technion" + - given-names: "Tom" + family-names: "Yotam" + affiliation: "Code Metal" + - given-names: "Le" + family-names: "Chen" + affiliation: "Argonne National Laboratory" + - given-names: "Tomer" + family-names: "Bitan" + affiliation: "Technion" + - given-names: "Niranjan" + family-names: "Hasabnis" + affiliation: "Code Metal" + - given-names: "Gal" + family-names: "Oren" + affiliation: "Stanford University, Technion" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..e6e3a84e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,67 @@ +# Contributing to ParBench + +ParBench is a bring-your-own-suite harness, so the most valuable contribution is a new +benchmark suite. Bug reports on existing specs are the next most valuable. Both are welcome. + +## Proposing a new suite + +Read [docs/adding-a-suite.md](docs/adding-a-suite.md) first: it walks the full path from +source layout to a first verified run. Open a +[new suite issue](https://github.com/Scientific-Computing-Lab/ParBench/issues/new?template=new-suite.yml) +before writing code, so the pinning and licensing questions are settled early. + +What a suite contribution must satisfy: + +- **One spec per kernel variant**, validating against `schema/spec_schema.json`. The naming + rule is `unique_id` = `{source_suite}-{kernel_name}-{parallel_api}`, and the file is + `specs/{unique_id}.json`. +- **A pinned upstream source.** `provenance` records the repository URL, the exact commit, + and the license. ParBench does not vendor benchmark source; it fetches it at a pinned + commit, and suites keep their upstream licenses. +- **Run arguments read from the source, not the docs.** Read the argument parser + (`grep -n argc `) before writing `run.args`. Two bundled kernels were silently + broken for weeks by documentation-derived arguments. Check your specs with + `python3 scripts/spec_tools/check_spec_argc.py --all`. +- **The strongest correctness oracle the benchmark supports.** `file_hash` or `file_diff` on + a computed output is strong, `numeric_comparison` against a declared tolerance is medium, + and `stdout_pattern` plus `exit_code` is weak: it proves execution, not numerical + correctness. Declare the strength in `verification.oracle_strength`. A weak oracle is + acceptable when the upstream references genuinely disagree across APIs, but say so in the + spec, and guard trivially satisfiable patterns with a `stdout_exclude_pattern`. +- **A verified baseline.** `python3 -m harness verify specs/.json` must pass on + the pristine reference before the spec enters an evaluation. A PASS against an unverified + baseline proves nothing. +- **One appended line per spec in `manifest.jsonl`.** The manifest is append-only: add lines, + never edit or remove existing ones. If a spec is retired, its manifest entry stays. +- **A pair contract** for cross-API pairs (`schema/pair_contract_schema.json`), declaring the + target-side run arguments and oracle. See `config/final_pair_contracts.json` for examples. + +## Reporting a spec or verify failure + +Open a +[bug report](https://github.com/Scientific-Computing-Lab/ParBench/issues/new?template=bug-report.yml) +with the spec's `unique_id`, your +platform and compiler versions, the exact command you ran, and its full output. Build and +verify failures are usually toolchain-specific, so the compiler version and OS matter as much +as the error text. + +Some failures are already known and are excluded from evaluation batches by code +(`EXCLUDED_SPECS` in `harness/constants.py`). Check that list before filing. + +## Before opening a pull request + +```bash +bash scripts/run_public_tests.sh # the test suite scoped to this release +python3 scripts/validate_schema.py --spec specs/.json # exits 0 when valid +python3 -m harness verify specs/.json # baseline verification +``` + +`python3 scripts/validate_schema.py --all` is the full-corpus check. It exits nonzero on any +checkout that does not have every benchmark tree fetched, and it always reports three errors +for each of the five retired manifest entries, so read its error list rather than its exit +status. + +Pull requests target `main` and are merged after review by the maintainers. Use the +[pull request template](.github/PULL_REQUEST_TEMPLATE.md) and keep the diff scoped to one +suite or one fix. Do not include machine-specific paths, credentials, or local +`config/paths.json` values in a contribution. diff --git a/README.md b/README.md index f4ba1864..59f0a272 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,8 @@ The five bundled suites below are a starting corpus, not a boundary. Any paralle can join the harness by writing one spec file per kernel variant against `schema/spec_schema.json` and appending one line to `manifest.jsonl`. Translation pairs, prompt payloads, and verification then work for the new suite exactly as they do for the -bundled ones. See **[docs/adding-a-suite.md](docs/adding-a-suite.md)** for the full path. +bundled ones. See **[docs/adding-a-suite.md](docs/adding-a-suite.md)** for the full path and +**[CONTRIBUTING.md](CONTRIBUTING.md)** for how to propose one. ## Benchmark corpus @@ -50,17 +51,21 @@ enables automatic discovery of translation pairs across APIs. Python 3.12 or later is required. ```bash -git clone +git clone https://github.com/Scientific-Computing-Lab/ParBench.git cd ParBench -python3 -m venv env_parbench -source env_parbench/bin/activate +python3 -m venv venv +source venv/bin/activate -# Core dependencies (harness, schema validation, augmentation) +# Exact pinned versions (the reproducible environment; recommended) +python3 -m pip install -r requirements-lock.txt + +# Or unpinned core dependencies (harness, schema validation, augmentation) python3 -m pip install -r requirements.txt -# Or for exact pinned versions (reproducible environment) -python3 -m pip install -r requirements-lock.txt +# Install the project itself, so `python3 -m harness` and the analysis +# scripts resolve from any directory +python3 -m pip install -e . ``` Optional dependency groups from `pyproject.toml`: @@ -73,19 +78,24 @@ python3 -m pip install ".[all]" # Everything ``` Building and running kernels additionally requires compilers for the target APIs (`nvcc` for -CUDA, `g++` with `-fopenmp` for OpenMP, OpenCL headers and runtime for OpenCL). Tested +CUDA, `g++` with `-fopenmp` for OpenMP, OpenCL headers and runtime for OpenCL). On Ubuntu, +the OpenMP path needs only `sudo apt-get install build-essential` (`g++` and `make`). Tested versions are listed in `config/compiler_inventory.txt`. ## Quick start +Run these from the repository root, with the virtual environment from **Installation** above +active. + ```bash -source env_parbench/bin/activate +source venv/bin/activate # 1. Fetch the Rodinia sources (the only submodule; ~101 MB) git submodule update --init rodinia +ln -s . rodinia/rodinia-src # the specs address the tree as rodinia/rodinia-src -# 2. Validate the manifest and all specs -python3 scripts/validate_schema.py --all +# 2. Validate one spec against the schema (exits 0 on a fresh clone) +python3 scripts/validate_schema.py --spec specs/rodinia-nw-omp.json # 3. Build, run, and verify one kernel (OpenMP — needs only a multi-core CPU; # nw generates its own input matrix, so no data download is required) @@ -95,8 +105,23 @@ python3 -m harness verify specs/rodinia-nw-omp.json python3 -m harness pairs ``` +Step 1's symlink is required, not optional: the submodule checks out at `rodinia/`, while +every Rodinia spec declares its `repo_root` as `rodinia/rodinia-src`. Without the link, +validation reports a missing source directory for every Rodinia spec, and `harness verify` +stops with "Working directory does not exist". + +`python3 scripts/validate_schema.py --all` is the full-corpus check, and it is **expected to +exit nonzero on a fresh clone**: it reports missing source files for every benchmark tree you +have not fetched yet, plus three errors for each of the five retired manifest entries +described under [Validation](#validation). Run it once the trees you care about are in place, and read its +error list rather than its exit code. + Two platform notes. (1) OpenMP kernels need a compiler with `-fopenmp` (GNU g++; Apple's -clang on macOS does not support it — use Linux, the tested platform). (2) Rodinia kernels +clang on macOS does not support it — use Linux, the tested platform). The bundled OpenMP +Makefiles hardcode `CC = g++` and the harness passes no compiler override, so whatever is +named `g++` on your PATH must be a real GNU g++; on macOS that means putting Homebrew's gcc +ahead of Apple's clang shim, and there is no flag that points the harness at a differently +named compiler. (2) Rodinia kernels that read input files (bfs, hotspot, srad, ...) additionally need the separate [Rodinia data package](https://rodinia.cs.virginia.edu/), unpacked to `rodinia/rodinia-src/data/`; kernels with self-generated inputs (nw, lud, pathfinder, @@ -113,10 +138,12 @@ pre-registered eligibility rules, leaving 2,160 valid records over 136 translati three models). These JSON and CSV files carry every number in the paper: pass@k tables, direction asymmetry, augmentation trends, the failure taxonomy, and per-suite results. -Raw per-task records are not tracked in this repository. The self-contained reproducibility -artifact (Docker image recipe, raw records, and `reproduce.sh`) is published as a release -asset; **[docs/reproducing-paper.md](docs/reproducing-paper.md)** explains both reproduction -paths. +Raw per-task records are not tracked in this repository. They ship instead inside the +self-contained reproducibility artifact (Docker image recipe, raw records, and +`reproduce.sh`), published as the asset `parbench-artifact-neurips2026.zip` on the +[neurips2026-artifact release](https://github.com/Scientific-Computing-Lab/ParBench/releases/tag/neurips2026-artifact). +**[artifact/README.md](artifact/README.md)** documents that archive and +**[docs/reproducing-paper.md](docs/reproducing-paper.md)** explains both reproduction paths. ## Project structure @@ -134,7 +161,7 @@ ParBench/ ├── results/analysis/final/ # Canonical aggregate results (the paper's numbers) ├── expected_outputs/ # Reference outputs for bit-exact table verification ├── artifact/ # Reproducibility artifact (Dockerfile, reproduce.sh) -└── config/ # Machine-specific config (git-ignored paths.json) +└── config/ # Pair contracts and paths.json (tracked, sanitized defaults) ``` ## Spec anatomy @@ -169,11 +196,13 @@ python3 scripts/validate_schema.py --all # everything ``` The validator checks schema conformance, `unique_id` naming and format, API consistency, that -every listed file exists on disk, and the prompt/verification separation above. Two classes -of errors are expected and explained by the validator: (1) specs whose benchmark tree you -have not fetched yet report missing source files (fetch the tree, per the table above, and -they clear); (2) five phantom manifest entries remain permanently (the manifest is -append-only and retains entries whose spec files were deleted). +every listed file exists on disk, and the prompt/verification separation above. A single-spec +run exits 0. The `--all` run does not, because two classes of errors are expected and are +explained in its output: (1) specs whose benchmark tree you have not fetched yet report +missing source files (fetch the tree, per the table above, and they clear); (2) five retired +manifest entries remain permanently, contributing three errors each (the manifest is append-only, so +it retains entries whose spec files were deleted). Judge `--all` by its error list, not by its +exit status. ## Requirements @@ -185,10 +214,14 @@ append-only and retains entries whose spec files were deleted). ## Testing ```bash +bash scripts/run_public_tests.sh # unit suite minus tests whose subjects are not in this release +python3 -m pytest tests/ # full suite (some tests need files outside this release) python3 -m pytest c_augmentation/test_transforms.py -v # augmentation transform tests -python3 scripts/validate_schema.py --all # schema validation +python3 scripts/validate_schema.py --all # schema validation (see above on its exit status) ``` +`pytest` comes from the `[dev]` dependency group. + ## License MIT (see `LICENSE`). The bundled benchmark suites are fetched from their upstream @@ -196,7 +229,9 @@ repositories and keep their own upstream licenses; ParBench does not vendor them ## Citation -The ParBench paper — "ParBench: A Kernel-Centric Benchmark for Evaluating LLM-Based Parallel -Code Translation" (NeurIPS 2026) — describes the benchmark design and the evaluation results. -We will add the citation entry and the arXiv link to this section when the camera-ready -version is published. +To cite the software, use `CITATION.cff` in the repository root; GitHub renders it as a +ready-to-paste BibTeX or APA entry under "Cite this repository". It also carries the +preferred citation for the paper, "ParBench: A Kernel-Centric Benchmark for Evaluating +LLM-Based Parallel Code Translation" (NeurIPS 2026), which describes the benchmark design and +the evaluation results. The arXiv link and DOI will be added to `CITATION.cff` when the +camera-ready version is published. diff --git a/artifact/README.md b/artifact/README.md index 28cddfc4..2fcad2c9 100644 --- a/artifact/README.md +++ b/artifact/README.md @@ -2,10 +2,21 @@ Self-contained artifact for reproducing all tables and figures in the NeurIPS 2026 paper "ParBench: A Kernel-Centric Benchmark for Evaluating LLM-Based Parallel Code Translation." +## Where the artifact lives + +The artifact is a single release asset, `parbench-artifact-neurips2026.zip` (43,070,656 bytes, +SHA-256 `e7fce335ab60ba7273e9cb6449f4563823b397af4af77954d2acd1d35e31fd66`), attached to the +[neurips2026-artifact release](https://github.com/Scientific-Computing-Lab/ParBench/releases/tag/neurips2026-artifact). +It is not tracked in the repository, because it carries the raw per-task evaluation records +that the repository itself does not ship. + ## Quick Start ```bash -tar xf parbench-artifact-v1.tar.gz +curl -L -O https://github.com/Scientific-Computing-Lab/ParBench/releases/download/neurips2026-artifact/parbench-artifact-neurips2026.zip +sha256sum parbench-artifact-neurips2026.zip # macOS: shasum -a 256 +# expect e7fce335ab60ba7273e9cb6449f4563823b397af4af77954d2acd1d35e31fd66 +unzip parbench-artifact-neurips2026.zip cd parbench-artifact docker build -t parbench . docker run --rm -v $(pwd)/output:/app/output parbench ./reproduce.sh @@ -15,29 +26,64 @@ docker run --rm -v $(pwd)/output:/app/output parbench ./reproduce.sh ## Without Docker -If you prefer running without Docker (or are reviewing from a cloned repository): +Same archive, no container. Run from the unpacked `parbench-artifact/` directory: ```bash -# 1. Enter the project root -cd - -# 2. Create and activate a Python 3.12+ virtual environment +# 1. Create and activate a Python 3.12+ virtual environment python3 -m venv venv source venv/bin/activate -# 3. Install pinned dependencies and the project package +# 2. Install pinned dependencies and the project package pip install -r requirements-lock.txt pip install -e . -# 4. Run reproduction (~10-15 minutes) +# 3. Run reproduction (about a minute on a modern laptop; longer on first pip install) bash artifact/reproduce.sh -# 5. Check output +# 4. Check output ls output/ # expect 35 files: 5 .tex tables + 15 figures × (PNG + PDF) ``` **Note:** Table values (`.tex` files) are deterministic across platforms. Figure appearance may vary slightly due to font availability and matplotlib backend differences. +## From a cloned repository + +A clone of the public repository is **not** enough to run `reproduce.sh`. The script rebuilds +every table and figure from the raw per-task records under `results/evaluation/`, and those +records are not tracked in the repository - they exist only inside the release archive above. +Running `bash artifact/reproduce.sh` in a clone exits 1 during Step 1 with +`ERROR: Results directory not found: .../results/evaluation/together-qwen-3.5-397b-a17b`. + +What a clone alone does give you: + +- `results/analysis/final/` - the committed canonical aggregates of the 2026-08-13 build + (2,344 records, 184 excluded by the pre-registered eligibility rules, 2,160 valid). Every + number printed in the paper comes from these JSON, CSV and Markdown files, so they can be + read and cross-checked directly without running anything. +- `expected_outputs/` - the reference `.tex` tables and `.pdf` figures, for diffing against a + reproduction run made from the archive. +- The benchmark itself: `specs/`, `manifest.jsonl`, `schema/`, `harness/`, and the analysis + scripts. + +To reproduce end to end from a clone, download the archive and copy its `results/evaluation/` +tree into the clone, then run the script from the clone: + +```bash +git clone https://github.com/Scientific-Computing-Lab/ParBench.git +curl -L -O https://github.com/Scientific-Computing-Lab/ParBench/releases/download/neurips2026-artifact/parbench-artifact-neurips2026.zip +unzip parbench-artifact-neurips2026.zip +cp -r parbench-artifact/results/evaluation ParBench/results/evaluation +cd ParBench +python3 -m venv venv && source venv/bin/activate +pip install -r requirements-lock.txt && pip install -e . +bash artifact/reproduce.sh +``` + +This path is verified: it exits 0 and writes 35 files to `output/` (5 `.tex` tables and 15 +figures as both PDF and PNG), plus 14 regenerated aggregates under `output/analysis/`. +The script resolves its project root from the location of `manifest.jsonl`, so it does not +matter whether you run it from the clone or from the unpacked archive. + ## What This Reproduces ### Tables (LaTeX, deterministic) @@ -101,7 +147,7 @@ parbench-artifact/ ## What's Included - **Raw evaluation results** (~97 MB): 2,344 per-task JSON files from 3 models (Qwen 3.5 397B-A17B, GPT-5.4, GPT-5.3-codex) across 6 translation directions and 5 augmentation levels (2,160 valid records after excluding the 13 correctness-ineligible specs: 10 KNOWN_FAIL plus 3 mixbench performance-only) -- **Generated translation source**: each result JSON carries the model's output in its `translated_files` field, a mapping of `{filename: source-code-string}` for every file the model was asked to translate. 2,341 of the 2,344 records contain generated source (the 3 omissions are GPT-5.4 extraction failures on the myocyte kernel family; in the shipped replay records they carry top-level `overall_status: "NOT_REPLAYABLE"` with the original verdict preserved as `parent.overall_status: "EXTRACTION_FAIL"`, and Table T1 reports them in the Extract column). To flatten these into a browsable tree plus a `translations_manifest.jsonl` (one line per record, keyed by `namespace` and `model`, with a SHA-256 per file), run `python3 scripts/rebuttal/export_translations.py --project-root . --out-dir `. +- **Generated translation source**: each result JSON carries the model's output in its `translated_files` field, a mapping of `{filename: source-code-string}` for every file the model was asked to translate. 2,341 of the 2,344 records contain generated source (the 3 omissions are GPT-5.4 extraction failures on the myocyte kernel family; in the shipped replay records they carry top-level `overall_status: "NOT_REPLAYABLE"` with the original verdict preserved as `parent.overall_status: "EXTRACTION_FAIL"`, and Table T1 reports them in the Extract column). The records are plain JSON, so the field can be read directly with any JSON tool; the internal helper that flattens it into a browsable tree is not part of this distribution. - **Analysis scripts**: Full pipeline from raw results to paper tables/figures - **Kernel specs**: 206 JSON spec files defining the benchmark's translation tasks - **Docker environment**: Exact Python dependency pins for bit-for-bit table reproduction @@ -123,10 +169,25 @@ parbench-artifact/ ## Verifying Outputs **Tables (deterministic):** Diff the generated `.tex` files against `expected_outputs/`. -Equality holds for the evaluation corpus the artifact builders stage under -`results/evaluation/` (the sealed replay namespace with corrected verdicts, the same -records behind the paper's canonical `results/analysis/final/` build); a reproduction -over the superseded pre-replay records will differ. + +Equality is conditional on the evaluation corpus you ran over. `expected_outputs/` was built +from the sealed replay namespace, the corrected-verdict records behind the paper's canonical +`results/analysis/final/` aggregates. A reproduction over the superseded pre-replay records +runs to completion but produces different pass rates. To tell which corpus you have, compare +this run's regenerated aggregates against the committed canonical ones, which every table in +the paper is printed from. Run this from a repository clone, which is where +`results/analysis/final/` lives: + +```bash +python3 -c " +import json +for d in ('output/analysis', 'results/analysis/final'): + p = f'{d}/quantitative_findings_azure-gpt-5.4.json' + print(d, json.load(open(p))['canonical']['aggregate_pass_rates']['overall']['value']) +" +``` + +Matching values mean you are on the sealed corpus and the `.tex` diffs below should be empty. ```bash diff output/t1_overall_pass.tex expected_outputs/t1_overall_pass.tex diff --git a/artifact/croissant.json b/artifact/croissant.json index 6e135c55..823c82d2 100644 --- a/artifact/croissant.json +++ b/artifact/croissant.json @@ -5,10 +5,19 @@ "column": "cr:column", "conformsTo": "dct:conformsTo", "cr": "http://mlcommons.org/croissant/", - "data": {"@id": "cr:data", "@type": "@json"}, - "dataType": {"@id": "cr:dataType", "@type": "@vocab"}, + "data": { + "@id": "cr:data", + "@type": "@json" + }, + "dataType": { + "@id": "cr:dataType", + "@type": "@vocab" + }, "dct": "http://purl.org/dc/terms/", - "examples": {"@id": "cr:examples", "@type": "@json"}, + "examples": { + "@id": "cr:examples", + "@type": "@json" + }, "extract": "cr:extract", "field": "cr:field", "fileObject": "cr:FileObject", @@ -41,20 +50,64 @@ "name": "ParBench Kernel Specifications", "description": "206 JSON specification files defining parallel code translation tasks across CUDA, OpenMP, OpenCL, and OpenMP target offload APIs, sourced from 5 HPC benchmark suites (Rodinia, HeCBench, XSBench, RSBench, mixbench).", "license": "https://opensource.org/licenses/MIT", - "url": "https://anonymous.4open.science/r/parbench_artifact_neurips/", + "url": "https://github.com/Scientific-Computing-Lab/ParBench", "version": "1.0.0", - "keywords": ["parallel computing", "code translation", "CUDA", "OpenMP", "OpenCL", "benchmark", "LLM evaluation"], + "keywords": [ + "parallel computing", + "code translation", + "CUDA", + "OpenMP", + "OpenCL", + "benchmark", + "LLM evaluation" + ], "datePublished": "2026-05-06", "conformsTo": "http://mlcommons.org/croissant/1.1", - "citeAs": "Anonymous. ParBench: A Meta-Benchmark for LLM-Based Parallel Code Translation. NeurIPS 2026 Evaluations & Datasets Track (under review).", + "citeAs": "Samyak Jhaveri, Erel Kaplan, Tom Yotam, Le Chen, Tomer Bitan, Niranjan Hasabnis, and Gal Oren. ParBench: A Benchmark for Reliable Evaluation of LLM Parallel Code Translation. NeurIPS 2026.", + "creator": [ + { + "@type": "sc:Person", + "name": "Samyak Jhaveri", + "affiliation": "University of California, Irvine" + }, + { + "@type": "sc:Person", + "name": "Erel Kaplan", + "affiliation": "Technion" + }, + { + "@type": "sc:Person", + "name": "Tom Yotam", + "affiliation": "Code Metal" + }, + { + "@type": "sc:Person", + "name": "Le Chen", + "affiliation": "Argonne National Laboratory" + }, + { + "@type": "sc:Person", + "name": "Tomer Bitan", + "affiliation": "Technion" + }, + { + "@type": "sc:Person", + "name": "Niranjan Hasabnis", + "affiliation": "Code Metal" + }, + { + "@type": "sc:Person", + "name": "Gal Oren", + "affiliation": "Stanford University, Technion" + } + ], "distribution": [ { "@type": "cr:FileSet", "name": "kernel-specs", "description": "206 JSON specification files, one per kernel-API variant, defining build, run, and verification configurations for parallel code translation tasks", "includes": "specs/*.json", - "encodingFormat": "application/json", - "sha256": "7548c3677b45e45158b879c5c3fc132b81219e7129e8874fcf0b56b785bf5a25" + "encodingFormat": "application/json" }, { "@type": "cr:FileObject", @@ -62,14 +115,14 @@ "description": "JSON Schema (draft-07) defining the spec file structure", "contentUrl": "schema/spec_schema.json", "encodingFormat": "application/schema+json", - "sha256": "d69539482601409889b87fae36e44d565e79a7f1950efbe6262d4fa42fa05cfa" + "sha256": "418bc1f16d15b1c98edf9f4abe72493b418938f1869cb8a2529fe513cd39382d" } ], "rai:dataCollection": "Specifications curated from existing open-source HPC benchmark suites (Rodinia, HeCBench, XSBench, RSBench, mixbench). Each spec is manually authored by the benchmark developers to define build, run, and verification configurations. No crowdsourcing, scraping, or automated data collection was used.", "rai:dataCollectionType": "Curated from existing sources", "rai:dataUseCases": "Evaluating the correctness of LLM-generated parallel code translations (e.g., CUDA to OpenMP). The specs define ground-truth build and verification pipelines against which translated code is tested. Not intended for training language models.", - "rai:dataLimitations": "Covers only four parallel APIs (CUDA, OpenMP, OpenCL, OpenMP target offload). Does not include SYCL, HIP, Kokkos, or other parallel frameworks. Benchmark kernels are drawn from scientific computing workloads and may not represent all parallel programming patterns. Verification oracles vary in strength (2 strong, 5 medium, 46 weak, remainder untagged).", - "rai:dataBiases": "Kernel selection is biased toward scientific computing and numerical simulation workloads from established HPC benchmark suites. Rodinia contributes the majority of specs (60/206). Underrepresented domains include graph analytics, machine learning training kernels, and I/O-bound workloads.", + "rai:dataLimitations": "Covers only four parallel APIs (CUDA, OpenMP, OpenCL, OpenMP target offload). Does not include SYCL, HIP, Kokkos, or other parallel frameworks. Benchmark kernels are drawn from scientific computing workloads and may not represent all parallel programming patterns. Verification oracles vary in strength: of the 206 specs, 2 declare a strong oracle, 5 medium, and 48 weak, while 151 carry no oracle_strength tag.", + "rai:dataBiases": "Kernel selection is biased toward scientific computing and numerical simulation workloads from established HPC benchmark suites. HeCBench contributes the largest share of specs (135 of 206) and Rodinia the second largest (60 of 206). Underrepresented domains include graph analytics, machine learning training kernels, and I/O-bound workloads.", "rai:personalSensitiveInformation": "No personal or sensitive data. All specs describe computational kernels operating on synthetic or public scientific datasets. No human subjects, demographic data, or personally identifiable information is present.", "rai:dataSocialImpact": "ParBench enables evaluation of automated code translation tools, which could accelerate software portability across GPU and CPU platforms. Potential positive impact: reducing the cost of maintaining parallel codebases. No known negative social impacts from the benchmark data itself.", "rai:dataPreprocessingProtocol": "Kernel source files are used as-is from upstream repositories at pinned commits. No preprocessing, filtering, or transformation is applied to the source code. Spec JSON files are authored manually following a JSON Schema (draft-07) contract.", diff --git a/docs/benchmark-tree-pins.md b/docs/benchmark-tree-pins.md index 8edc9d21..fe0dc947 100644 --- a/docs/benchmark-tree-pins.md +++ b/docs/benchmark-tree-pins.md @@ -1,27 +1,100 @@ # Benchmark tree pins and fetch recipes -> Moved here 2026-08-16 from a retired onboarding doc (full text in git history). -> This is the standing home for the pinned SHAs and the two fetch recipes. +ParBench does not vendor benchmark source. +Each suite is fetched at a pinned commit, so every spec's `provenance.repo_root` resolves to the same bytes the paper measured. +Only `rodinia` is a git submodule; the other four trees are gitignored working copies that the recipes below create. -## Pinned SHAs +## Pinned commits -| tree | pinned SHA | upstream | -|---|---|---| -| `rodinia` | `9c10d3ea16ddba2ba057cc3951a9efc4c2cc18a4` | `github.com/yuhc/gpu-rodinia` | -| `mixbench/mixbench-src` | `32edeca98bdd63b32769e3c7460676b9fd567f06` | `github.com/ekondis/mixbench` | -| `xsbench/xsbench-src` | `ba08e5221af6106252b866e50ea123c69d31a4e2` | `github.com/ANL-CESAR/XSBench` | -| `rsbench/rsbench-src` | `34b644787ea9af4fb188e1253da72e09bbed9989` | `github.com/ANL-CESAR/RSBench` | -| `HeCBench-master` | `22785cdd708de5dca56525a277b31fe119171fd1` | `github.com/zjin-lcf/HeCBench` | +| Suite | Directory | Pinned commit | Upstream | +|---|---|---|---| +| Rodinia | `rodinia/rodinia-src` | `9c10d3ea16ddba2ba057cc3951a9efc4c2cc18a4` | `github.com/yuhc/gpu-rodinia` | +| HeCBench | `HeCBench-master` | `22785cdd708de5dca56525a277b31fe119171fd1` | `github.com/zjin-lcf/HeCBench` | +| mixbench | `mixbench/mixbench-src` | `32edeca98bdd63b32769e3c7460676b9fd567f06` | `github.com/ekondis/mixbench` | +| XSBench | `xsbench/xsbench-src` | `ba08e5221af6106252b866e50ea123c69d31a4e2` | `github.com/ANL-CESAR/XSBench` | +| RSBench | `rsbench/rsbench-src` | `34b644787ea9af4fb188e1253da72e09bbed9989` | `github.com/ANL-CESAR/RSBench` | -Known deviation (accepted 2026-08-16): the Mac's `HeCBench-master` working copy is at -`accf77fde427525ea9a270a7be58a9b56eb36169`, not the pin; the Linux experiment machine -is at the pin. Owner ruling: leave the Mac copy as is. +Rodinia, mixbench, XSBench, and RSBench specs record the same commit under `provenance.repository.commit`, and the two agree. +HeCBench specs mostly record an `archive-download` provenance instead (129 of 135; the other 6 carry the pinned commit) - the pin in this table is the authoritative one. +`git rev-parse HEAD` inside a fetched tree is the verification step: it must print the commit in this table. -## Recipe 1: attach history to a tree whose FILES are already present +Sizes, so you can pick what to fetch: Rodinia ~101 MB, HeCBench ~1.2 GB, and the three small suites a few MB each. +A Rodinia-only checkout is enough for the 60 Rodinia specs and for all analysis work. -`--mixed` rewrites only the index, never the working tree. Use for a copied tree that -arrived without `.git`. Guard with `[ -e "$1/.git" ]`, never `git rev-parse --git-dir` -(rev-parse walks UP into the parbench repo and the guard silently no-ops). +## Rodinia (the one submodule) + +Run from the repository root. + +```bash +git submodule update --init rodinia +git -C rodinia rev-parse HEAD # must print 9c10d3ea16ddba2ba057cc3951a9efc4c2cc18a4 + +# Specs address Rodinia as rodinia/rodinia-src (provenance.repo_root), but the +# submodule checks out at rodinia/. This symlink bridges the two; without it no +# Rodinia spec resolves its source files. +ln -sfn . rodinia/rodinia-src +``` + +Then apply the two shipped patches. +Both are tracked under `patches/` and both apply cleanly onto the pinned checkout. + +```bash +# Dry run first: --check changes nothing and reports whether the patch applies. +git -C rodinia apply --check -p1 "$PWD/patches/rodinia-build-fixes.patch" +git -C rodinia apply --check -p1 "$PWD/patches/rodinia-hotspot-correctness.patch" + +git -C rodinia apply -p1 "$PWD/patches/rodinia-build-fixes.patch" +git -C rodinia apply -p1 "$PWD/patches/rodinia-hotspot-correctness.patch" +``` + +`rodinia-build-fixes.patch` carries toolchain fixes for nine files (`common/make.config`, four CUDA makefiles, four OpenCL makefiles). +One fix is toolchain-specific: it sets `CUDA_DIR = /opt/nvidia/hpc_sdk/Linux_x86_64/24.3/cuda/12.3` in `common/make.config`, the reference machine's NVIDIA HPC SDK path. +If your CUDA lives elsewhere, apply the patch and then edit that one variable to your CUDA root; nothing else in the patch is machine-specific. +CPU-only work (OpenMP kernels such as `rodinia-nw-omp`) never reads `CUDA_DIR`, so the default is harmless there. +`rodinia-hotspot-correctness.patch` restores a missing `else` branch in `openmp/hotspot/hotspot_openmp.cpp` so `delta` is never reused stale. + +`git apply --check` doubles as the idempotence test: on an already-patched tree the forward check fails and `--check --reverse` succeeds. +If neither direction applies, the tree has diverged and needs hand inspection - do not force the patch. + +Expect `rodinia` to show as dirty on a build host afterwards: it carries these patches plus some checked-in binaries. +The submodule is configured with `ignore = dirty`, so that dirt does not surface in the parent repository's `git status`. + +## The other four suites + +These are gitignored working copies, so no git command in the parent repository brings them in. +Fetch each at its pinned commit, from the repository root. + +```bash +fetch_pinned() { # fetch_pinned + mkdir -p "$1" + git -C "$1" init -q . + git -C "$1" remote add origin "$2" 2>/dev/null || true + git -C "$1" fetch -q --depth 1 origin "$3" + git -C "$1" checkout -q FETCH_HEAD + echo "$1 -> $(git -C "$1" rev-parse HEAD)" +} + +fetch_pinned mixbench/mixbench-src https://github.com/ekondis/mixbench 32edeca98bdd63b32769e3c7460676b9fd567f06 +fetch_pinned xsbench/xsbench-src https://github.com/ANL-CESAR/XSBench ba08e5221af6106252b866e50ea123c69d31a4e2 +fetch_pinned rsbench/rsbench-src https://github.com/ANL-CESAR/RSBench 34b644787ea9af4fb188e1253da72e09bbed9989 + +# HeCBench keeps its *.tar.bz datasets in Git LFS, and the upstream repository is +# currently over its GitHub LFS budget, so the smudge filter fails and the +# checkout aborts partway. Skipping the smudge leaves those files as pointer +# stubs and lets the checkout finish. No ParBench spec reads a .tar.bz, so the +# stubs cost nothing. +GIT_LFS_SKIP_SMUDGE=1 fetch_pinned HeCBench-master https://github.com/zjin-lcf/HeCBench 22785cdd708de5dca56525a277b31fe119171fd1 +``` + +Each call echoes the resolved `HEAD`; check it against the table above. +Fetching an arbitrary commit by SHA works because GitHub serves reachable non-tip commits; `--depth 1` keeps the download to one commit. + +Fetching at the pinned SHAs dirties nothing in the ParBench repository itself. + +## Attaching history to a tree whose files are already present + +Use this when a tree arrived as a plain copy, with source files but no `.git`. +`git reset --mixed` rewrites only the index, never the working tree, so the copied files survive. ```bash attach() { # attach @@ -33,20 +106,6 @@ attach() { # attach } ``` -## Recipe 2: materialize a tree that is ABSENT - -A checkout, not a mixed reset. Used on the Mac 2026-08-16 for the three small suites. - -```bash -mkdir -p && cd -git init -q . && git remote add origin -git fetch -q --depth 1 origin -git checkout -q FETCH_HEAD -``` - -Expect `rodinia` to show as dirty afterwards on build hosts: it carries the documented -toolchain patches (`docs/rodinia_toolchain_patches.diff`; re-apply with -`scripts/spec_tools/apply_rodinia_patches.sh`) plus some checked-in binaries. -The other four trees are gitignored working copies; rodinia is the one submodule -(its dirt is suppressed by `submodule.rodinia.ignore=dirty`). A fetch at the pinned -SHAs dirties nothing in the parbench repo. +Guard the check with `[ -e "$dir/.git" ]`, never `git -C "$dir" rev-parse --git-dir`. +`rev-parse` walks up the directory tree, finds the enclosing ParBench repository, and returns 0 for a tree that has no history of its own, which makes the guard silently skip every call. +Use `-e` rather than `-d`: a submodule's `.git` is a file, not a directory. diff --git a/docs/running-evals.md b/docs/running-evals.md index 34ea128a..e9d3c117 100644 --- a/docs/running-evals.md +++ b/docs/running-evals.md @@ -52,7 +52,7 @@ rather than by convention: - `EXCLUDED_SPECS` (10 specs): kernels whose baseline is broken in the current toolchain (KNOWN_FAIL). The batch runner's `resolve_excluded_specs` - (`scripts/evaluation/run_eval_batch.py:78`) removes them from every batch on entry; + (`scripts/evaluation/run_eval_batch.py:118`) removes them from every batch on entry; the CLI flag `--excluded-specs` can add exclusions but never remove one. Rationale: source-side, an unverified baseline makes a PASS meaningless; target-side, broken infrastructure makes the evaluation unfair. diff --git a/pyproject.toml b/pyproject.toml index efff9e7e..b6ee6ecd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,8 @@ eval = [ analysis = [ "matplotlib>=3.9", "numpy>=1.26", + "scipy>=1.14", + "scienceplots>=2.2", ] dev = [ "pytest>=8.0", diff --git a/requirements.txt b/requirements.txt index 4d61c90d..b2dca400 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,7 @@ openai>=1.50.0 matplotlib>=3.9 scienceplots>=2.2 numpy>=1.26 +scipy>=1.14 # Dev — testing and linting pytest>=8.0 diff --git a/scripts/check_generated_registry.py b/scripts/check_generated_registry.py deleted file mode 100644 index 3ae6a983..00000000 --- a/scripts/check_generated_registry.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 -"""Validate .claude/generated-outputs.tsv, the registry read by generated-file-guard.sh. - -The guard fails open on infrastructure problems, so a registry that has quietly rotted -would disable protection with no signal. This script is that signal. It is a mandatory -universal check in /validate. - -Checks, per row: - 1. exactly 3 tab-separated fields - 2. mode is literally `block` or `warn` (no CR, no padding, no case variance) - the - guard blocks a malformed row rather than degrading it to `warn`, so a bad mode - turns into a surprise block at edit time - 3. every generator path named in the row exists on disk - 4. the glob matches at least one git-tracked file (a row matching nothing is either - a typo or a stale entry, and both read as protection that is not there) - -And file-level: - 5. no CR anywhere (a CRLF editor on the macOS half of this project) - 6. the file ends with a newline (`while read` drops an unterminated final line, which - is exactly the line an "add a generator" commit appends) - -Exit 0 = clean, 1 = problems found (printed to stdout, one per line). - -`--uncovered` additionally lists git-tracked files under the directories the registry -already reaches into that NO row covers. That is advisory, not a failure: it is the -worklist for closing coverage gaps, not a claim that every file there must be registered. - -`--fail-uncovered` (Task 3, gate E-03) is the enforcing sibling: an uncovered file -under `--dirs` is a FAILURE (exit 1). It scans tracked AND untracked files; with -`--changed-only` the scan narrows to files changed by the current work (git status: -staged, modified, or untracked). A supplied directory that is missing or empty is -tolerated — e.g. results/augmentation_final, whose producing task is deferred. -""" - -from __future__ import annotations - -import argparse -import fnmatch -import subprocess -import sys -from pathlib import Path - -ROOT = Path(__file__).resolve().parent.parent -VALID_MODES = {"block", "warn"} - - -def tracked_files(root: Path) -> list[str]: - out = subprocess.run( - ["git", "-C", str(root), "ls-files"], - capture_output=True, text=True, check=True, - ) - return out.stdout.splitlines() - - -def untracked_files(root: Path) -> list[str]: - out = subprocess.run( - ["git", "-C", str(root), "ls-files", "--others", "--exclude-standard"], - capture_output=True, text=True, check=True, - ) - return out.stdout.splitlines() - - -def changed_files(root: Path) -> list[str]: - """Files changed by the current work: staged, modified, or untracked.""" - out = subprocess.run( - ["git", "-C", str(root), "status", "--porcelain", - "--untracked-files=all"], - capture_output=True, text=True, check=True, - ) - paths: list[str] = [] - for line in out.stdout.splitlines(): - if len(line) < 4: - continue - path = line[3:] - # Renames are shown as "old -> new"; the new path is the changed one. - if " -> " in path: - path = path.split(" -> ", 1)[1] - paths.append(path) - return paths - - -def in_dirs(path: str, dirs: list[str]) -> bool: - return any(path.startswith(d.rstrip("/") + "/") for d in dirs) - - -def parse_rows(raw: str) -> list[tuple[int, str]]: - """Return (1-based line number, line) for non-comment, non-blank lines.""" - rows = [] - for i, line in enumerate(raw.split("\n"), start=1): - stripped = line.strip() - if not stripped or stripped.startswith("#"): - continue - rows.append((i, line)) - return rows - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("--uncovered", action="store_true", - help="also list tracked files under --dirs that no row covers") - ap.add_argument("--fail-uncovered", action="store_true", - help="enforcing mode: an uncovered tracked-or-untracked file " - "under --dirs is a failure (exit 1); missing or empty " - "supplied directories are tolerated") - ap.add_argument("--changed-only", action="store_true", - help="with --fail-uncovered: scan only files changed by the " - "current work (staged, modified, or untracked)") - ap.add_argument("--dirs", nargs="+", - default=["results/analysis", "results/augmentation"], - help="directories the --uncovered advisory scans " - "(default: the two named in CLAUDE.md invariant 2)") - ap.add_argument("--root", type=Path, default=ROOT, - help="repository root (default: this checkout; used by tests)") - args = ap.parse_args() - - root = args.root.resolve() - registry = root / ".claude" / "generated-outputs.tsv" - - if not registry.exists(): - print(f"FAIL: registry missing: {registry}") - return 1 - - # read_bytes, not read_text: text mode applies universal-newline translation, which - # silently rewrites \r\n to \n and makes the CR check below unable to ever fire. - raw = registry.read_bytes().decode("utf-8") - problems: list[str] = [] - - if "\r" in raw: - problems.append("FAIL: registry contains CR characters; the guard would see a " - "mode of 'block\\r' and treat the row as malformed") - if not raw.endswith("\n"): - problems.append("FAIL: registry does not end with a newline; the guard's read " - "loop would drop the final row") - - files = tracked_files(root) - globs: list[str] = [] - - for lineno, line in parse_rows(raw): - fields = line.split("\t") - if len(fields) != 3: - problems.append(f"FAIL: line {lineno}: expected 3 tab-separated fields, got {len(fields)}") - continue - - glob, generator, mode = fields - globs.append(glob) - - if mode not in VALID_MODES: - problems.append(f"FAIL: line {lineno}: mode {mode!r} is not 'block' or 'warn'") - - # A row may name more than one generator, joined with '+'. - for gen in (g.strip() for g in generator.split("+")): - if not gen: - problems.append(f"FAIL: line {lineno}: empty generator name") - elif not (root / gen).exists(): - problems.append(f"FAIL: line {lineno}: generator does not exist: {gen}") - - if not any(fnmatch.fnmatch(f, glob) for f in files): - problems.append(f"FAIL: line {lineno}: glob matches no tracked file: {glob}") - - for p in problems: - print(p) - - if args.uncovered: - uncovered = [ - f for f in files - if in_dirs(f, args.dirs) - and not any(fnmatch.fnmatch(f, g) for g in globs) - ] - if uncovered: - print(f"\nADVISORY: {len(uncovered)} tracked files under registered directories " - f"are covered by no row:") - for f in uncovered: - print(f" {f}") - - if args.fail_uncovered: - # Tracked plus untracked; --changed-only narrows to current work. - if args.changed_only: - candidates = changed_files(root) - else: - candidates = files + untracked_files(root) - # A missing or empty supplied directory is fine (a deferred producing - # task, e.g. results/augmentation_final); only files that exist can be - # uncovered. - enforce_uncovered = sorted({ - f for f in candidates - if in_dirs(f, args.dirs) - and not any(fnmatch.fnmatch(f, g) for g in globs) - }) - if enforce_uncovered: - print(f"\nFAIL: {len(enforce_uncovered)} file(s) under enforced " - f"directories are covered by no registry row:") - for f in enforce_uncovered: - print(f" {f}") - problems.extend(f"uncovered: {f}" for f in enforce_uncovered) - - if problems: - print(f"\n{len(problems)} problem(s) found.") - return 1 - - print(f"Registry OK: {len(globs)} rows, all globs match tracked files, " - f"all generators exist.") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/pre-commit.sh b/scripts/pre-commit.sh deleted file mode 100755 index 9fcc4b8f..00000000 --- a/scripts/pre-commit.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -# Native git pre-commit hook body (wired at .git/hooks/pre-commit). -# Runs the two fast deterministic checks and fails the commit if either fails. -# Replaces the retired PreToolUse .validation_passed sentinel gate (2026-08-14): -# a native git hook is lighter, runs inside git, and does not block every Bash call. -# For the heavier project-health pass (schema, tests, manifest), run /validate on demand. -set -euo pipefail -ROOT="$(git rev-parse --show-toplevel)" -cd "$ROOT" -python3 .claude/hooks/check_stale_counts.py || { echo "pre-commit: stale-count check failed"; exit 1; } -python3 scripts/check_generated_registry.py || { echo "pre-commit: generated-registry check failed"; exit 1; } -exit 0 diff --git a/scripts/run_public_tests.sh b/scripts/run_public_tests.sh new file mode 100755 index 00000000..fcf82b41 --- /dev/null +++ b/scripts/run_public_tests.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Public-tree test runner: the unit suite minus tests whose subject script or +# fixture data is not part of the public release. CI (.github/workflows/ci.yml), +# CONTRIBUTING.md, and the PR template all call this script, so the exclusion +# list lives in exactly one place. On a full private checkout, plain +# `python3 -m pytest tests/` remains the stricter superset. +set -euo pipefail +cd "$(dirname "$0")/.." + +EXCLUDE=( + # subject scripts/spec_tools/apply_rodinia_patches.sh is not in the public release (10/10 tests) + --ignore=tests/test_apply_rodinia_patches.py + # needs scripts/build_artifact.sh and results/analysis/agentic_protocol_dry_run.json (3/3 tests) + --ignore=tests/test_artifact_protocol_deliverables.py + # needs the raw results/evaluation/ record tree (1/1 test) + --ignore=tests/test_emit_release_inventory.py + + # fixtures cite the evidence file results/analysis/pair_witnesses.json + --deselect tests/test_freeze_final_contract.py::TestHappyPath + # needs the results/analysis/negative_controls/ records + --deselect tests/test_mixbench_scope.py::TestNegativeControlEvidence + + # subject scripts/lib/artifact_release_common.sh is not in the public release + --deselect tests/test_artifact_release_common.py::test_parse_all_flags_and_extras + --deselect tests/test_artifact_release_common.py::test_parse_defaults_empty + --deselect tests/test_artifact_release_common.py::test_parse_missing_value_returns_2 + --deselect tests/test_artifact_release_common.py::test_guard_absent_output_ok + --deselect tests/test_artifact_release_common.py::test_guard_empty_output_ok + --deselect tests/test_artifact_release_common.py::test_guard_nonempty_no_resume_refuses + --deselect tests/test_artifact_release_common.py::test_guard_resume_without_sidecar_refuses + --deselect tests/test_artifact_release_common.py::test_guard_resume_matching_provenance_ok + --deselect tests/test_artifact_release_common.py::test_guard_resume_mismatch_refuses + --deselect tests/test_artifact_release_common.py::test_scratch_trap_removes_only_owned_dir + + # subject scripts/orchestration/gate_verdict.sh is not in the public release + --deselect "tests/test_gate_verdict.py::TestGateVerdict::test_genuine_pass[gate-verdict]" + --deselect "tests/test_gate_verdict.py::TestGateVerdict::test_genuine_pass[task-result]" + --deselect "tests/test_gate_verdict.py::TestGateVerdict::test_missing_log_is_exit_2[gate-verdict]" + --deselect "tests/test_gate_verdict.py::TestGateVerdict::test_missing_log_is_exit_2[task-result]" + --deselect "tests/test_gate_verdict.py::TestGateVerdict::test_empty_log_is_exit_2[gate-verdict]" + --deselect "tests/test_gate_verdict.py::TestGateVerdict::test_empty_log_is_exit_2[task-result]" + + # subject scripts/batch/run_phase3.sh is not in the public release + --deselect tests/test_eval_exclusions.py::test_run_phase3_has_no_copied_exclusion_list + --deselect tests/test_eval_exclusions.py::test_run_phase3_batch_and_retry_invocations_rely_on_automatic_exclusions + + # needs Rodinia sources on disk; no suite tree is assumed here + --deselect tests/test_check_spec_argc_contracts.py::test_frozen_v1_registry_still_shows_the_two_streamcluster_findings +) + +python3 -m pytest tests/ -q "${EXCLUDE[@]}" "$@" diff --git a/tests/test_generated_registry_enforcement.py b/tests/test_generated_registry_enforcement.py deleted file mode 100644 index 27f37786..00000000 --- a/tests/test_generated_registry_enforcement.py +++ /dev/null @@ -1,123 +0,0 @@ -"""Task 3 (gate E-03): enforcing --fail-uncovered mode for the generated-file registry. - -check_generated_registry.py gains --fail-uncovered: any tracked-or-untracked file -changed by the current work under the caller-supplied directories that no registry -row covers is a failure (exit 1), while --uncovered stays advisory (exit 0). -A supplied directory that is missing or empty (e.g. results/augmentation_final, -whose producing task is deferred) is tolerated, never an error. -""" -from __future__ import annotations - -import subprocess -import sys -from pathlib import Path - -import pytest - -PROJECT_ROOT = Path(__file__).resolve().parents[1] -SCRIPT = PROJECT_ROOT / "scripts" / "check_generated_registry.py" - - -def run_script(root: Path, *args: str) -> subprocess.CompletedProcess: - return subprocess.run( - [sys.executable, str(SCRIPT), "--root", str(root), *args], - capture_output=True, text=True, - ) - - -def git(root: Path, *args: str) -> None: - subprocess.run(["git", "-C", str(root), *args], check=True, - capture_output=True) - - -@pytest.fixture() -def repo(tmp_path: Path) -> Path: - root = tmp_path / "repo" - (root / ".claude").mkdir(parents=True) - (root / "scripts").mkdir() - (root / "scripts" / "gen.py").write_text("# generator\n") - (root / "results" / "analysis").mkdir(parents=True) - (root / "results" / "analysis" / "covered.json").write_text("{}\n") - (root / ".claude" / "generated-outputs.tsv").write_text( - "results/analysis/covered.json\tscripts/gen.py\tblock\n" - ) - git(root, "init", "-q") - git(root, "config", "user.email", "t@t") - git(root, "config", "user.name", "t") - git(root, "add", "-A") - git(root, "commit", "-qm", "init") - return root - - -def test_clean_repo_passes(repo: Path) -> None: - proc = run_script(repo, "--fail-uncovered", "--changed-only", - "--dirs", "results/analysis") - assert proc.returncode == 0, proc.stdout + proc.stderr - - -def test_uncovered_untracked_file_fails(repo: Path) -> None: - (repo / "results" / "analysis" / "new_output.json").write_text("{}\n") - proc = run_script(repo, "--fail-uncovered", "--changed-only", - "--dirs", "results/analysis") - assert proc.returncode == 1 - assert "new_output.json" in proc.stdout - - -def test_uncovered_modified_tracked_file_fails(repo: Path) -> None: - stray = repo / "results" / "analysis" / "stray.md" - stray.write_text("old\n") - git(repo, "add", "-A") - git(repo, "commit", "-qm", "add stray") - stray.write_text("changed by current work\n") - proc = run_script(repo, "--fail-uncovered", "--changed-only", - "--dirs", "results/analysis") - assert proc.returncode == 1 - assert "stray.md" in proc.stdout - - -def test_changed_only_ignores_unchanged_uncovered_files(repo: Path) -> None: - stray = repo / "results" / "analysis" / "stray.md" - stray.write_text("committed, not changed since\n") - git(repo, "add", "-A") - git(repo, "commit", "-qm", "add stray") - proc = run_script(repo, "--fail-uncovered", "--changed-only", - "--dirs", "results/analysis") - assert proc.returncode == 0 - proc2 = run_script(repo, "--fail-uncovered", "--dirs", "results/analysis") - assert proc2.returncode == 1 - - -def test_covered_changed_file_passes(repo: Path) -> None: - (repo / "results" / "analysis" / "covered.json").write_text('{"new": 1}\n') - proc = run_script(repo, "--fail-uncovered", "--changed-only", - "--dirs", "results/analysis") - assert proc.returncode == 0 - - -def test_missing_and_empty_dirs_are_tolerated(repo: Path) -> None: - (repo / "results" / "empty_dir").mkdir() - proc = run_script( - repo, "--fail-uncovered", "--changed-only", - "--dirs", "results/analysis", "results/augmentation_final", - "results/empty_dir", - ) - assert proc.returncode == 0, proc.stdout + proc.stderr - - -def test_uncovered_flag_stays_advisory(repo: Path) -> None: - """--uncovered keeps its pre-Task-3 compatibility contract: it lists - uncovered *tracked* files and never fails on them.""" - (repo / "results" / "analysis" / "new_output.json").write_text("{}\n") - git(repo, "add", "-A") - git(repo, "commit", "-qm", "add uncovered output") - proc = run_script(repo, "--uncovered", "--dirs", "results/analysis") - assert proc.returncode == 0 - assert "new_output.json" in proc.stdout - - -def test_real_registry_still_validates() -> None: - proc = subprocess.run( - [sys.executable, str(SCRIPT)], capture_output=True, text=True, - cwd=PROJECT_ROOT, - ) - assert proc.returncode == 0, proc.stdout + proc.stderr