Add public web benchmarks with control, xwebmcp and Playwright arms

.github/workflows/_run.yml

@@ -22,6 +22,21 @@ on:
         required: false
         default: "10"
         type: string
+      attempts:
+        description: Trials per task and arm
+        required: false
+        default: "1"
+        type: string
+      runner:
+        description: Runner preset from runners/, or empty for arm defaults
+        required: false
+        default: ""
+        type: string
+      model:
+        description: Model override for every selected arm
+        required: false
+        default: ""
+        type: string
       toolset_ref:
         description: Git ref for toolsets built from source
         required: false
@@ -65,6 +80,10 @@ jobs:
         with:
           bun-version: "1.3.3"
 
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: "22"
+
       - name: Install
         run: |
           uv sync --frozen
@@ -78,14 +97,19 @@ jobs:
           ARMS: ${{ inputs.arms }}
           TASKS: ${{ inputs.tasks }}
           CONCURRENCY: ${{ inputs.concurrency }}
+          ATTEMPTS: ${{ inputs.attempts }}
+          RUNNER: ${{ inputs.runner }}
+          MODEL: ${{ inputs.model }}
           TOOLSET_REF: ${{ inputs.toolset_ref }}
         run: |
           run_id="$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
           jobs_dir="$RUNNER_TEMP/jobs"
           echo "run_file=$jobs_dir/run-$run_id.json" >>"$GITHUB_OUTPUT"
           echo "experiment=$BENCHMARK_NAME-$run_id" >>"$GITHUB_OUTPUT"
-          args=(--tasks "$TASKS" --concurrency "$CONCURRENCY" --jobs-dir "$jobs_dir" --run-id "$run_id")
+          args=(--tasks "$TASKS" --concurrency "$CONCURRENCY" --attempts "$ATTEMPTS" --jobs-dir "$jobs_dir" --run-id "$run_id")
           [[ -n "$ARMS" ]] && args+=(--arms "$ARMS")
+          [[ -n "$RUNNER" ]] && args+=(--runner "$RUNNER")
+          [[ -n "$MODEL" ]] && args+=(--model "$MODEL")
           [[ -n "$TOOLSET_REF" ]] && args+=(--toolset-ref "$TOOLSET_REF")
           uv run bench run "$BENCHMARK_NAME" "${args[@]}"
 

.github/workflows/ci.yml

@@ -18,13 +18,25 @@ jobs:
       - uses: astral-sh/setup-uv@b75a909f75acd358c2196fb9a5f1299a9a8868a4 # v6.7.0
         with:
           version: "0.8.17"
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+        with:
+          node-version: "22"
       - run: uv sync --frozen
       - run: uv run ruff check .
       - run: uv run ruff format --check .
       - run: uv run pytest -q
+      - run: node --test suites/public-web/*.test.mjs
       - name: Prepare every static arm for one task
         run: uv run bench prepare kernel-mcp-clawbench --arms codex-kernel-mcp,claude-kernel-mcp --tasks v2-1134-chapter-finder-redcross --out "$RUNNER_TEMP/prepared"
 
+      - name: Prepare all public-web arms
+        run: uv run bench prepare public-web --out "$RUNNER_TEMP/public-web"
+      - name: Prepare alternate runners
+        run: |
+          for runner in claude pi; do
+            uv run bench prepare public-web --runner "$runner" --tasks github-merged-pr --out "$RUNNER_TEMP/public-web-$runner"
+          done
+
   report:
     runs-on: ubuntu-latest
     defaults:

.github/workflows/public-web.yml

@@ -0,0 +1,41 @@
+name: Public web browser approaches
+
+on:
+  workflow_dispatch:
+    inputs:
+      tasks:
+        description: Comma-separated task IDs, or all
+        default: all
+      arms:
+        description: Comma-separated arm names, or empty for all three
+        default: ""
+      runner:
+        description: Agent runner preset (pi keeps one agent loop across providers)
+        type: choice
+        options: [codex, claude, pi]
+        default: codex
+      model:
+        description: Optional model override (Pi uses provider/model)
+        default: ""
+      attempts:
+        description: Trials per task and arm
+        default: "5"
+      concurrency:
+        description: Concurrent control/treatment pairs, and concurrent Playwright-only trials
+        default: "5"
+
+permissions:
+  contents: read
+
+jobs:
+  run:
+    uses: ./.github/workflows/_run.yml
+    with:
+      benchmark: public-web
+      tasks: ${{ inputs.tasks }}
+      arms: ${{ inputs.arms }}
+      runner: ${{ inputs.runner }}
+      model: ${{ inputs.model }}
+      attempts: ${{ inputs.attempts }}
+      concurrency: ${{ inputs.concurrency }}
+    secrets: inherit

README.md

@@ -28,6 +28,7 @@ A full ClawBench run is 129 trials per arm. [Preparing tasks](#quick-start) for
 | **Suite** | A set of tasks, adapted into Harbor tasks, with its own scoring. | [ClawBench](https://github.com/TIGER-AI-Lab/ClawBench): 129 tasks on real websites, scored by intercepting the final request |
 | **Toolset** | What the agent uses to drive the browser. | `kernel-mcp-local`: the [Kernel MCP server](https://github.com/kernel/kernel-mcp-server), built from source |
 | **Agent** | A Harbor agent, model, and version. | Codex with `gpt-5.6-luna` |
+| **Runner preset** | Reusable agent, model, version and agent options, independent of browser approach. | `pi`, `claude`, `codex` in `runners/` |
 | **Arm** | One agent + toolset + browser configuration. The thing being compared. | `codex-kernel-mcp-site-config` |
 | **Configurator** | Optional. Picks browser settings per task, and can add advice for the agent. | `site-config`: asks Kernel's [config registry](https://kernel.sh/docs/config-registry) what works on the task's site |
 | **Benchmark** | A suite plus the arms to run on it. One GitHub workflow each. | `kernel-mcp-clawbench` |
@@ -72,9 +73,9 @@ uv run bench run kernel-mcp-clawbench \
 
 Harbor writes one job directory per arm under `jobs/`, containing rewards, trajectories, and verifier output. To summarize jobs, run `bun report/report.ts --arm <name>=<job dir>` (needs [Bun](https://bun.sh)).
 
-`--tasks` takes comma-separated task IDs or `all`. `--concurrency` sets concurrent trials per arm. `--toolset-ref` builds a toolset from a specific git ref, e.g. a Kernel MCP server branch.
+`--tasks` takes comma-separated task IDs or `all`. `--concurrency` sets concurrent trials per arm. `--toolset-ref` builds a toolset from a specific git ref, e.g. a Kernel MCP server branch. `--runner` selects a preset from `runners/` and `--model` overrides its model, for every selected arm. Both also work with `bench prepare`; a model override alone does not change the runner.
 
-The run also writes `jobs/run-<id>.json`, which maps each arm to its job directory, exit status, and the browser settings and site guidance each task got.
+The run also writes `jobs/run-<id>.json`, which maps each arm to its job directory, exit status, resolved runner/model/version/options, and the browser settings and site guidance each task got. Use a unique `--run-id` for each experiment; an existing run manifest is never overwritten.
 
 A run of two arms on two tasks (Red Cross and Indeed) produced this summary:
 
@@ -119,6 +120,126 @@ The Kernel MCP server on ClawBench.
 
 ClawBench's primary score is the lenient reward. `kernel_mcp_valid` records whether the agent called Kernel MCP's `execute_playwright_code` and used its assigned browser on every call, so a failed task can be told apart from a trial that never used the toolset.
 
+### `public-web`
+
+Compare three browser approaches on nine signed-out tasks across six sites. The tasks cover
+repository facts and search, hotel rules, flight search and cart preparation, retail search,
+and comparing game prices. No logins, payment details or purchases. One task,
+`github-latest-release`, is a placebo: the treatment arm gets GitHub's tools, but none of them
+reads a release, so a difference between control and treatment there is noise.
+
+| Arm | Browser tools | Injected tools |
+|---|---|---|
+| `public-control` | Playwright, persistent browser REPL, computer actions and native WebMCP | None |
+| `public-treatment` | Same as control | The site's eligible xwebmcp tools |
+| `public-playwright` | Playwright code only | None |
+
+All three use the same selected runner/model (Codex by default), stealth browser settings, task prompt,
+initial page, fixtures and grading. Control and treatment have identical browser instructions; the
+Playwright-only instruction prohibits WebMCP, REPL, computer actions, raw CDP and direct HTTP.
+The stdio proxy filters the advertised tools and rejects calls outside the arm's allowlist,
+calls on another session, WebMCP registration/removal and, in the Playwright-only arm, code
+that refers to `webmcp`. A rejected call never reaches the browser, so it is reported and does
+not fail the trial. The proxy also withholds the server's own usage instructions, which rank
+WebMCP first and name tools the arms lack, and in the Playwright-only arm cuts the WebMCP
+guidance from the tool's descriptions, so that arm is not told to use what it cannot. This
+is not an arbitrary-code sandbox: instructions and trajectory review still matter for calls
+that bypass MCP.
+
+The catalog, task definitions, eligible tool batches and grader come from one exact
+`kernel/xwebmcp` revision in [suites/public-web/source.toml](suites/public-web/source.toml).
+Task-specific development-branch references are not followed. Kernel MCP is independently
+pinned in the three arm files. This runs in this repository's Harbor harness; it does not
+run xwebmcp's development-eval runner or depend on an unmerged PR.
+
+Prepare all arms without credentials, then smoke-test one task before a full paid run:
+
+```bash
+uv run bench prepare public-web --out /tmp/public-web
+uv run bench run public-web --tasks github-merged-pr --attempts 1 --concurrency 1
+gh workflow run public-web.yml -f tasks=all -f attempts=5 -f concurrency=5
+```
+
+#### Switching models and providers
+
+The browser arms contain no model-specific configuration. Their default is `runner = "codex"`;
+`--runner` replaces the agent, version and **all** agent options together, so Codex options cannot
+leak into another agent. `--model` changes only the model ID, leaving the runner's version and
+options fixed. The workflow exposes the same runner selector and optional model field.
+
+| Preset | Agent | Version | Default model |
+|---|---|---|---|
+| `codex` | Codex | 0.163.0 | `gpt-6-luna` |
+| `claude` | Claude Code | 2.1.301 | `claude-sonnet-5` |
+| `pi` | Pi | 0.85.1 | `anthropic/claude-sonnet-5` |
+
+For model comparisons, use **Pi in both runs** so the agent loop, version, browser tools and
+`thinking = "medium"` setting stay fixed. Pi uses `provider/model` IDs and the appropriate provider
+key (`ANTHROPIC_API_KEY` or `OPENAI_API_KEY`). Run close together with the same tasks and settings:
+
+```bash
+uv run bench run public-web --runner pi --model anthropic/claude-sonnet-5 \
+  --run-id sonnet-5-pilot --tasks github-merged-pr --attempts 1 --concurrency 1
+uv run bench run public-web --runner pi --model openai/gpt-5.4 \
+  --run-id gpt-54-pilot --tasks github-merged-pr --attempts 1 --concurrency 1
+```
+
+To benchmark the native Claude Code agent instead, use `--runner claude --model claude-sonnet-5`.
+Comparing native Codex against Claude Code changes both the model and the agent; it is not a
+model-only comparison. The same thinking label also does not imply equal compute across models.
+
+Pi is pinned to the version supported by the pinned Harbor MCP adapter. Use IDs available in that
+Pi model catalog; newer model IDs may require a compatible Pi/Harbor update. No provider login or
+model access is implied by selecting a preset, and live model availability still depends on the
+provider account. Add another MCP-capable Harbor agent by adding a file under `runners/`, rather
+than duplicating the three browser arms. Credentials stay in environment variables, not presets.
+
+The full workflow defaults to 135 trials (9 tasks × 3 arms × 5 attempts), each with a fresh
+browser. Control and treatment differ only in the injected tools, so `benchmarks/public-web.toml`
+pairs them: for every task and attempt, one control and one treatment trial run back to back,
+in an order that is random but balanced within each task and recorded in the run manifest.
+`--concurrency` is then the number of pairs running at once. The Playwright-only arm changes
+more than the tools, so it stays one independent Harbor job in the same window. `--unpaired`
+runs all three arms that way, which is faster and gives no paired comparison. Do not compare
+different run dates.
+
+Two limits come from xwebmcp's own runner. Its turn cap becomes a cap on browser calls, which
+the call proxy enforces for every agent: once a trial has made that many, further calls are
+refused and the agent is told to finalize. Its per-trial spend cap, `max_cost_usd` in
+[suites/public-web/source.toml](suites/public-web/source.toml), is applied by the report,
+because Harbor's agents cannot all stop on spend: a trial that cost more is not a pass. It
+only applies to trials whose model cost Harbor could price. Each task's timeout and limits
+are in the run manifest.
+
+The agent calls `finalize` with its JSON answer. The verifier applies the pinned answer
+schema/rules and, for the cart task, reads the final page independently and compares it to
+the answer. That page check covers the route, cabin, passenger count and amount due, not the
+flight numbers or whether the fare is the cheapest. The rules and the grader are uploaded only
+for verification, so the agent's container never holds them. A success claim alone is
+insufficient, and a trial with no permitted browser call gets zero reward. The report's
+headline is the checked pass rate: passes on the tasks graded by page end state or pinned
+rules, with a 95% interval. Passes on shape/range tasks are reported apart as format-only,
+because they show a well-formed, in-range answer and nothing about whether it is true. Even
+pinned rules do not verify every returned fact. Audit examples before publishing correctness
+claims; this catalog is a screening set, not a held-out validation set.
+
+Reports show per-arm and per-task results, agent-only time (excluding setup and grading),
+input/cached/output tokens, browser call counts, toolset validity, rejected calls and recorded
+model cost with pricing coverage. For the paired arms they add, per task, how many pairs passed
+in both arms or in one only, the speedup over both-pass pairs with a sign test, and whether the
+two answers agree where the task defines that. Tool use is split into injected and native
+tools, and into trials that tried one and trials where an invocation succeeded. What stopped
+trials that did not pass is listed by cause: bot blocks, site or Kernel errors, an unreadable
+page, a used-up call cap, overspending, and errors. A trial is errored, not failed, when it
+never got a fair run: a failed setup, or a model provider that rejected or rate-limited the
+agent. Errored trials stay in every denominator and mark the run incomplete.
+
+The call proxy sees only MCP calls, so the report also reads each agent's trajectory and flags
+trials with a tool call that reached the task's site, the Kernel API or the task's files
+outside the arm's browser tools, such as `curl` from the shell. A flag does not change a score;
+review flagged trials before publishing. Raw artifacts stay local; the existing redacted
+Braintrust publisher also records source revisions and grading modes.
+
 ## Extending
 
 ### Add an arm
@@ -177,13 +298,14 @@ Create `suites/<name>/suite.py` with:
 - `preflight()`: checks external dependencies before any browser is created.
 - `generate(task_ids, out_dir)`: writes Harbor tasks and returns their directories. The tasks should run on the browser Harbor provides (`KERNEL_SESSION_ID`, `/harbor/kernel/connection.json`) and leave tool wiring to toolsets.
 
-Then add `benchmarks/<name>.toml` and a workflow that calls `_run.yml`.
+Then add `benchmarks/<name>.toml` and a workflow that calls `_run.yml`. To compare two arms that differ in one thing, list them in the benchmark as `pairs = [["baseline-arm", "candidate-arm"]]`: their trials then run as back-to-back matched pairs, and each trial directory gets a `pair.json` a report can match on.
 
 ## Layout
 
 ```
-benchmarks/      suite + arms per benchmark
-arms/            agent, model, toolset, browser settings
+benchmarks/      suite + arms per benchmark, and which arms run as matched pairs
+arms/            browser approach + runner preset (or inline agent settings)
+runners/         reusable agent, model, version and agent options
 suites/          task sets adapted to Harbor
 toolsets/        browser tools for the agent
 configurators/   per-task browser settings and guidance
@@ -196,7 +318,7 @@ tests/
 
 - **Live websites change.** Arms in one run share the same time window. Compare arms within a run rather than across days.
 - **The Harbor provider is pinned to a fork.** Harbor is pinned to a `kernel/harbor` commit until [harbor-framework/harbor#2122](https://github.com/harbor-framework/harbor/pull/2122) is released.
-- **Reporting assumes ClawBench.** The report and Braintrust publisher expect ClawBench-shaped rewards and step layout. They will be generalized when a second suite lands.
+- **Scoring is suite-specific.** ClawBench reports intercepted-request rewards; public-web reports answer-rule passes and page-checked passes separately. Neither report substitutes agent claims for its configured grader.
 
 ## License
 

arms/public-control.toml

@@ -0,0 +1,11 @@
+# Playwright, the browser REPL, computer actions and the site's own WebMCP tools, with no
+# xwebmcp tools injected. The baseline for public-treatment.
+runner = "codex"
+toolset = "browser-approach"
+toolset_ref = "df5a5e2f9d701aba1d7b6cc115f939448e840b48"
+
+[toolset_args]
+approach = "control"
+
+[browser]
+config = { stealth = true }

arms/public-playwright.toml

@@ -0,0 +1,10 @@
+# Kernel MCP's execute_playwright_code only: no WebMCP, REPL or computer actions.
+runner = "codex"
+toolset = "browser-approach"
+toolset_ref = "df5a5e2f9d701aba1d7b6cc115f939448e840b48"
+
+[toolset_args]
+approach = "playwright"
+
+[browser]
+config = { stealth = true }

arms/public-treatment.toml

@@ -0,0 +1,11 @@
+# Same as public-control, with the site's xwebmcp tools registered in the browser before the
+# agent starts.
+runner = "codex"
+toolset = "browser-approach"
+toolset_ref = "df5a5e2f9d701aba1d7b6cc115f939448e840b48"
+
+[toolset_args]
+approach = "treatment"
+
+[browser]
+config = { stealth = true }

benchmarks/public-web.toml

@@ -0,0 +1,6 @@
+# Signed-out tasks on public sites, comparing browser approaches under one runner: the hybrid
+# toolset with and without injected xwebmcp tools, and Playwright alone.
+suite = "public-web"
+arms = ["public-control", "public-treatment", "public-playwright"]
+# Control and treatment differ only in the injected tools, so their trials run as matched pairs.
+pairs = [["public-control", "public-treatment"]]