From c5cb1cc709f496233802af774f639e2ae7b824fc Mon Sep 17 00:00:00 2001 From: andrewleesteele <8799863+andrewleesteele@users.noreply.github.com> Date: Thu, 5 Nov 2026 20:01:09 +0000 Subject: [PATCH 1/2] Add retail-cart-providers: the retail-cart tasks on competing providers' browsers A remote-browser environment runs the task container on local Docker and drives a Kernel, Browserbase, Steel, Hyperbrowser or Anchor browser over CDP, and a playwright-mcp toolset gives every arm the same tools, so arms differ only in the browser. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/_run.yml | 6 + .github/workflows/ci.yml | 2 + .github/workflows/retail-cart-providers.yml | 45 ++++++ README.md | 23 +++ arms/codex-pw-anchor.toml | 24 +++ arms/codex-pw-browserbase.toml | 21 +++ arms/codex-pw-hyperbrowser.toml | 14 ++ arms/codex-pw-kernel.toml | 14 ++ arms/codex-pw-steel.toml | 15 ++ benchmarks/retail-cart-providers.toml | 10 ++ harness/definitions.py | 13 +- harness/environment.py | 61 ++++++++ harness/providers.py | 144 ++++++++++++++++++ harness/run.py | 17 ++- suites/retail-cart/image/Dockerfile | 11 +- suites/retail-cart/image/bench/cdp-execute | 34 +++++ suites/retail-cart/image/bench/finalize-mcp | 4 +- suites/retail-cart/image/bench/open-start-url | 32 +++- suites/retail-cart/image/bench/score | 26 +++- tests/test_definitions.py | 22 +++ tests/test_environment.py | 72 +++++++++ tests/test_playwright_mcp.py | 20 +++ tests/test_providers.py | 86 +++++++++++ tests/test_run.py | 6 + toolsets/playwright-mcp/image/bin/browser-mcp | 6 + .../playwright-mcp/image/layer.Dockerfile | 3 + toolsets/playwright-mcp/toolset.py | 61 ++++++++ 27 files changed, 776 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/retail-cart-providers.yml create mode 100644 arms/codex-pw-anchor.toml create mode 100644 arms/codex-pw-browserbase.toml create mode 100644 arms/codex-pw-hyperbrowser.toml create mode 100644 arms/codex-pw-kernel.toml create mode 100644 arms/codex-pw-steel.toml create mode 100644 benchmarks/retail-cart-providers.toml create mode 100644 harness/providers.py create mode 100755 suites/retail-cart/image/bench/cdp-execute create mode 100644 tests/test_playwright_mcp.py create mode 100644 tests/test_providers.py create mode 100755 toolsets/playwright-mcp/image/bin/browser-mcp create mode 100644 toolsets/playwright-mcp/image/layer.Dockerfile create mode 100644 toolsets/playwright-mcp/toolset.py diff --git a/.github/workflows/_run.yml b/.github/workflows/_run.yml index e0ee4b5..106f2ea 100644 --- a/.github/workflows/_run.yml +++ b/.github/workflows/_run.yml @@ -65,6 +65,12 @@ jobs: CLAWBENCH_JUDGE_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} CLAWBENCH_JUDGE_MODEL: deepseek/deepseek-v4-pro CLAWBENCH_JUDGE_API_TYPE: openai-completions + # Other browser providers, for remote-browser arms. + BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} + BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} + STEEL_API_KEY: ${{ secrets.STEEL_API_KEY }} + HYPERBROWSER_API_KEY: ${{ secrets.HYPERBROWSER_API_KEY }} + ANCHOR_API_KEY: ${{ secrets.ANCHOR_API_KEY }} BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} BRAINTRUST_PROJECT: ${{ vars.BRAINTRUST_PROJECT }} steps: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72ff36b..828bfaa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,8 @@ jobs: run: uv run bench prepare kernel-mcp-clawbench --arms codex-kernel-mcp,claude-kernel-mcp --tasks v2-1134-chapter-finder-redcross --out "$RUNNER_TEMP/prepared" - name: Prepare the static retail-cart arm run: uv run bench prepare retail-cart --arms codex-repl --out "$RUNNER_TEMP/prepared-retail-cart" + - name: Prepare the retail-cart provider arms + run: uv run bench prepare retail-cart-providers --out "$RUNNER_TEMP/prepared-retail-cart-providers" report: runs-on: ubuntu-latest diff --git a/.github/workflows/retail-cart-providers.yml b/.github/workflows/retail-cart-providers.yml new file mode 100644 index 0000000..affec33 --- /dev/null +++ b/.github/workflows/retail-cart-providers.yml @@ -0,0 +1,45 @@ +name: Retail cart flows on each provider's browser + +on: + workflow_dispatch: + inputs: + arms: + description: Comma-separated arm names (empty runs every arm in benchmarks/retail-cart-providers.toml) + required: false + default: "" + type: string + tasks: + description: Comma-separated task IDs, or all + required: false + default: all + type: string + attempts: + description: Trials per task and arm + required: false + default: "10" + type: string + concurrency: + description: Concurrent trials per arm. Every arm's task containers run on the one runner. + required: false + default: "4" + type: string + playwright_mcp_version: + description: "@playwright/mcp version to install (empty installs the toolset's pinned version)" + required: false + default: "" + type: string + +permissions: + contents: read + +jobs: + benchmark: + uses: ./.github/workflows/_run.yml + secrets: inherit + with: + benchmark: retail-cart-providers + arms: ${{ inputs.arms }} + tasks: ${{ inputs.tasks }} + attempts: ${{ inputs.attempts }} + concurrency: ${{ inputs.concurrency }} + toolset_ref: ${{ inputs.playwright_mcp_version }} diff --git a/README.md b/README.md index 17942f6..10bb302 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ A run of two arms on two tasks (Red Cross and Indeed) produced this summary: | `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` | Codex and Claude Code arms; the `site-config` configurator's model | | `PURELY_MAIL_API_KEY`, `PURELY_MAIL_DOMAIN` | ClawBench creates a disposable [PurelyMail](https://purelymail.com) mailbox on your domain for every task, so agents can sign up and receive confirmation emails. | | `CLAWBENCH_JUDGE_BASE_URL`, `CLAWBENCH_JUDGE_API_KEY`, `CLAWBENCH_JUDGE_MODEL`, `CLAWBENCH_JUDGE_API_TYPE` | ClawBench's LLM judge, which checks the intercepted request against the task. Any OpenAI-compatible endpoint works with `CLAWBENCH_JUDGE_API_TYPE=openai-completions`; we use `deepseek/deepseek-v4-pro` through OpenRouter. See [ClawBench](https://github.com/TIGER-AI-Lab/ClawBench) for details. | +| `BROWSERBASE_API_KEY`, `BROWSERBASE_PROJECT_ID`, `STEEL_API_KEY`, `HYPERBROWSER_API_KEY`, `ANCHOR_API_KEY` | Arms on those providers' browsers (`environment = "remote-browser"`); checked before a run that includes them | | `BRAINTRUST_API_KEY`, `BRAINTRUST_PROJECT` | Optional: publishing results | ## Running in GitHub Actions @@ -139,6 +140,26 @@ Run several attempts per task to get a pass rate per site: uv run bench run retail-cart --attempts 10 ``` +### `retail-cart-providers` + +The `retail-cart` tasks on each browser provider's browser. Every arm uses the same agent (Codex, `gpt-6-luna`), the same tools ([Playwright MCP](https://github.com/microsoft/playwright-mcp) attached over CDP) and the same task container on the runner's Docker, so arms differ only in the browser. Each provider gets its own anti-bot settings: + +| Arm | Browser | +|---|---| +| `codex-pw-kernel` | Kernel, `stealth: true` | +| `codex-pw-browserbase` | Browserbase, Advanced Stealth (Scale plan), CAPTCHA solving, its residential proxies | +| `codex-pw-steel` | Steel, residential proxy and CAPTCHA solving | +| `codex-pw-hyperbrowser` | Hyperbrowser, stealth, proxy and CAPTCHA solving | +| `codex-pw-anchor` | Anchor, its US proxy, extra stealth and CAPTCHA solving | + +The start page, the agent and the verifier each connect to the browser separately, so sessions are created to outlive a CDP disconnect (Browserbase's `keepAlive`, Anchor's idle timeout). Scoring is the same as `retail-cart`. + +```bash +uv run bench run retail-cart-providers --attempts 10 --concurrency 4 +``` + +All arms' task containers run on one machine, so keep `--concurrency` low on a GitHub runner. + ## Extending ### Add an arm @@ -162,6 +183,8 @@ steering = true `toolset_args` are passed to the toolset; `kernel-mcp-local` takes `toolsets`, the Kernel MCP toolsets to enable: `playwright` (the default), `repl`, `computer`, or several. `agent_kwargs` are passed to the Harbor agent. +An arm with `environment = "remote-browser"` runs the task container on local Docker instead of in the Kernel browser's VM, and its `config` names a `provider` from `harness/providers.py` and that provider's session create `params`. The environment writes the browser's CDP URL to `/run/browser/cdp-url` in the container; the `playwright-mcp` toolset connects to it. + `config` takes any [`POST /browsers`](https://kernel.sh/docs/api-reference/browsers/create-a-browser-session) parameter, such as `stealth`, `proxy`, `region`, `extensions`, or `viewport`. Browsers are always headful. ### Add a configurator diff --git a/arms/codex-pw-anchor.toml b/arms/codex-pw-anchor.toml new file mode 100644 index 0000000..3ac96c3 --- /dev/null +++ b/arms/codex-pw-anchor.toml @@ -0,0 +1,24 @@ +# Codex driving an Anchor browser through Playwright MCP over CDP, with +# Anchor's anti-bot settings: its US proxy, extra stealth and CAPTCHA solving +# (both need the proxy). +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "anchor" + +[browser.config.params.session] +proxy = { active = true, type = "anchor_proxy", country_code = "us" } +# Minutes. The idle timeout covers the gaps between the start page, agent and +# verifier connections. +timeout = { max_duration = 45, idle_timeout = 30 } + +[browser.config.params.browser] +extra_stealth = { active = true } +captcha_solver = { active = true } diff --git a/arms/codex-pw-browserbase.toml b/arms/codex-pw-browserbase.toml new file mode 100644 index 0000000..18f379a --- /dev/null +++ b/arms/codex-pw-browserbase.toml @@ -0,0 +1,21 @@ +# Codex driving a Browserbase browser through Playwright MCP over CDP, with +# Browserbase's anti-bot settings: Advanced Stealth (Scale plan), CAPTCHA +# solving and its residential proxies. keepAlive keeps the session across the +# start page, agent and verifier connections. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "browserbase" + +[browser.config.params] +keepAlive = true +timeout = 2700 +proxies = true +browserSettings = { advancedStealth = true, solveCaptchas = true } diff --git a/arms/codex-pw-hyperbrowser.toml b/arms/codex-pw-hyperbrowser.toml new file mode 100644 index 0000000..3a6879c --- /dev/null +++ b/arms/codex-pw-hyperbrowser.toml @@ -0,0 +1,14 @@ +# Codex driving a Hyperbrowser browser through Playwright MCP over CDP, with +# Hyperbrowser's anti-bot settings: stealth, proxy and CAPTCHA solving. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "hyperbrowser" +params = { useStealth = true, useProxy = true, solveCaptchas = true, timeoutMinutes = 45 } diff --git a/arms/codex-pw-kernel.toml b/arms/codex-pw-kernel.toml new file mode 100644 index 0000000..bc9b4c4 --- /dev/null +++ b/arms/codex-pw-kernel.toml @@ -0,0 +1,14 @@ +# Codex driving a Kernel stealth browser through Playwright MCP over CDP, the +# same agent, tools and container as the other providers' codex-pw-* arms. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "kernel" +params = { stealth = true, timeout_seconds = 1800 } diff --git a/arms/codex-pw-steel.toml b/arms/codex-pw-steel.toml new file mode 100644 index 0000000..b78b0e6 --- /dev/null +++ b/arms/codex-pw-steel.toml @@ -0,0 +1,15 @@ +# Codex driving a Steel browser through Playwright MCP over CDP, with Steel's +# anti-bot settings: residential proxy and CAPTCHA solving. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "steel" +# timeout is in milliseconds. +params = { useProxy = true, solveCaptcha = true, timeout = 2700000 } diff --git a/benchmarks/retail-cart-providers.toml b/benchmarks/retail-cart-providers.toml new file mode 100644 index 0000000..61f9841 --- /dev/null +++ b/benchmarks/retail-cart-providers.toml @@ -0,0 +1,10 @@ +# The retail-cart tasks on each provider's browser, with the same agent, +# Playwright MCP tools and task container, so arms differ only in the browser. +suite = "retail-cart" +arms = [ + "codex-pw-kernel", + "codex-pw-browserbase", + "codex-pw-steel", + "codex-pw-hyperbrowser", + "codex-pw-anchor", +] diff --git a/harness/definitions.py b/harness/definitions.py index 57e12d4..00da835 100644 --- a/harness/definitions.py +++ b/harness/definitions.py @@ -11,6 +11,11 @@ from typing import Any ROOT = Path(__file__).resolve().parents[1] +# Harbor environment each arm `environment` runs its trials in. +ENVIRONMENTS = { + "kernel": "harness.environment:KernelTaskEnvironment", + "remote-browser": "harness.environment:RemoteBrowserEnvironment", +} @dataclass(frozen=True) @@ -26,6 +31,9 @@ class Arm: agent: str model: str toolset: str + # "kernel" runs the task container in the Kernel browser's VM; + # "remote-browser" runs it on local Docker and drives the browser over CDP. + environment: str = "kernel" agent_version: str | None = None agent_kwargs: dict[str, Any] = field(default_factory=dict) browser: Browser = field(default_factory=Browser) @@ -59,7 +67,10 @@ def load_benchmark(name: str) -> Benchmark: def load_arm(name: str) -> Arm: data = _read(ROOT / "arms" / f"{name}.toml") browser = data.pop("browser", {}) - return Arm(name=name, browser=Browser(**browser), **data) + arm = Arm(name=name, browser=Browser(**browser), **data) + if arm.environment not in ENVIRONMENTS: + raise SystemExit(f"Arm {name} has unknown environment {arm.environment!r}") + return arm def load_suite(name: str) -> ModuleType: diff --git a/harness/environment.py b/harness/environment.py index bf47afc..df773f3 100644 --- a/harness/environment.py +++ b/harness/environment.py @@ -20,6 +20,11 @@ the persona or WebGL renderer they ask for. `vm_setup` is a list of shell commands then run as root. A trial where either fails fails with `VmSetupError`. + +`RemoteBrowserEnvironment` instead runs the task container on local Docker +and drives a browser from any provider in `harness.providers` over CDP, so +providers are compared with the same agent, tools and container. Its config +file names the `provider` and that provider's session create `params`. """ from __future__ import annotations @@ -33,14 +38,20 @@ from pathlib import Path from typing import Any +from harbor.environments.docker.docker import DockerEnvironment from harbor.environments.kernel import KernelEnvironment from kernel import AsyncKernel, Kernel +from harness import providers + CONFIG_FILE = "kernel-config.json" PROXY_CREATE = "proxy_create" CHROMIUM_ENV = "chromium_env" VM_SETUP = "vm_setup" RUN_ID_ENV = "BENCH_RUN_ID" +# Where the task container finds the remote browser's CDP URL. Agents may not +# pass their environment on to the MCP servers they launch, so it is a file. +CDP_URL_FILE = "/run/browser/cdp-url" class ProxyCreationError(RuntimeError): @@ -51,6 +62,10 @@ class VmSetupError(RuntimeError): """The browser's VM could not be set up for the trial before the first page load.""" +class BrowserCreationError(RuntimeError): + """The provider could not create the trial's remote browser.""" + + SUPERVISOR_PROGRAM = """[program:chromium] command=/usr/bin/env {env} /usr/local/bin/chromium-launcher autostart=false @@ -210,3 +225,49 @@ async def stop(self, delete: bool) -> None: self._proxy_id = None except Exception as error: self.logger.error(f"Failed to delete Kernel proxy {self._proxy_id}: {error}") + + +class RemoteBrowserEnvironment(DockerEnvironment): + def __init__(self, environment_dir: Path, *args: Any, **kwargs: Any) -> None: + config = json.loads((Path(environment_dir) / CONFIG_FILE).read_text()) + self._provider_name: str = config["provider"] + self._provider = providers.get(self._provider_name) + self._create_params: dict[str, Any] = config.get("params") or {} + self._browser_session_id: str | None = None + super().__init__(environment_dir, *args, **kwargs) + + async def start(self, force_build: bool) -> None: + await super().start(force_build) + try: + session = await self._provider.create(self._create_params) + except Exception as error: + raise BrowserCreationError( + f"Could not create a {self._provider_name} browser: {error}" + ) from error + self._browser_session_id = session.id + self.logger.info(f"Created {self._provider_name} browser {session.id}") + self._persistent_env.update( + {"BROWSER_PROVIDER": self._provider_name, "BROWSER_SESSION_ID": session.id} + ) + directory = os.path.dirname(CDP_URL_FILE) + result = await self.exec( + f'mkdir -p {directory} && printf %s "$CDP_URL" > {CDP_URL_FILE} && chmod 0644 {CDP_URL_FILE}', + env={"CDP_URL": session.cdp_url}, + user="root", + ) + if result.return_code != 0: + raise BrowserCreationError(f"Could not write {CDP_URL_FILE}: {result.stderr}") + + async def stop(self, delete: bool) -> None: + try: + await super().stop(delete) + finally: + if self._browser_session_id is not None: + try: + await self._provider.release(self._browser_session_id) + self._browser_session_id = None + except Exception as error: + self.logger.error( + f"Failed to release {self._provider_name} browser " + f"{self._browser_session_id}: {error}" + ) diff --git a/harness/providers.py b/harness/providers.py new file mode 100644 index 0000000..9326bfa --- /dev/null +++ b/harness/providers.py @@ -0,0 +1,144 @@ +"""Cloud browser providers that a trial can drive over CDP. + +Each provider creates one browser session from an arm's create params and +releases it when the trial ends. The params are the provider's own session +create body, so an arm states exactly which stealth, proxy and CAPTCHA +settings it asked for. Sessions must outlive a CDP disconnect: the start page +is opened, the agent connects, and the verifier connects, each separately. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Any + +import httpx +from kernel import AsyncKernel + + +@dataclass(frozen=True) +class Session: + id: str + cdp_url: str + + +class Provider: + env: tuple[str, ...] = () + + async def create(self, params: dict[str, Any]) -> Session: + raise NotImplementedError + + async def release(self, session_id: str) -> None: + raise NotImplementedError + + +class KernelProvider(Provider): + env = ("KERNEL_API_KEY",) + + async def create(self, params: dict[str, Any]) -> Session: + async with AsyncKernel() as client: + browser = await client.browsers.create(**params) + return Session(browser.session_id, browser.cdp_ws_url) + + async def release(self, session_id: str) -> None: + async with AsyncKernel() as client: + await client.browsers.delete_by_id(session_id) + + +class HttpProvider(Provider): + base_url: str + key_header: str + + def _client(self) -> httpx.AsyncClient: + return httpx.AsyncClient( + base_url=self.base_url, + headers={self.key_header: os.environ[self.env[0]]}, + timeout=120, + ) + + +class BrowserbaseProvider(HttpProvider): + env = ("BROWSERBASE_API_KEY", "BROWSERBASE_PROJECT_ID") + base_url = "https://api.browserbase.com/v1" + key_header = "X-BB-API-Key" + + async def create(self, params: dict[str, Any]) -> Session: + body = {"projectId": os.environ["BROWSERBASE_PROJECT_ID"], **params} + async with self._client() as client: + response = await client.post("/sessions", json=body) + response.raise_for_status() + session = response.json() + return Session(session["id"], session["connectUrl"]) + + async def release(self, session_id: str) -> None: + body = {"projectId": os.environ["BROWSERBASE_PROJECT_ID"], "status": "REQUEST_RELEASE"} + async with self._client() as client: + (await client.post(f"/sessions/{session_id}", json=body)).raise_for_status() + + +class SteelProvider(HttpProvider): + env = ("STEEL_API_KEY",) + base_url = "https://api.steel.dev/v1" + key_header = "steel-api-key" + + async def create(self, params: dict[str, Any]) -> Session: + async with self._client() as client: + response = await client.post("/sessions", json=params) + response.raise_for_status() + session = response.json() + url = f"wss://connect.steel.dev?apiKey={os.environ['STEEL_API_KEY']}&sessionId={session['id']}" + return Session(session["id"], url) + + async def release(self, session_id: str) -> None: + async with self._client() as client: + (await client.post(f"/sessions/{session_id}/release")).raise_for_status() + + +class HyperbrowserProvider(HttpProvider): + env = ("HYPERBROWSER_API_KEY",) + base_url = "https://api.hyperbrowser.ai/api" + key_header = "x-api-key" + + async def create(self, params: dict[str, Any]) -> Session: + async with self._client() as client: + response = await client.post("/session", json=params) + response.raise_for_status() + session = response.json() + return Session(session["id"], session["wsEndpoint"]) + + async def release(self, session_id: str) -> None: + async with self._client() as client: + (await client.put(f"/session/{session_id}/stop")).raise_for_status() + + +class AnchorProvider(HttpProvider): + env = ("ANCHOR_API_KEY",) + base_url = "https://api.anchorbrowser.io/v1" + key_header = "anchor-api-key" + + async def create(self, params: dict[str, Any]) -> Session: + async with self._client() as client: + response = await client.post("/sessions", json=params) + response.raise_for_status() + session = response.json()["data"] + return Session(session["id"], session["cdp_url"]) + + async def release(self, session_id: str) -> None: + async with self._client() as client: + (await client.delete(f"/sessions/{session_id}")).raise_for_status() + + +PROVIDERS: dict[str, Provider] = { + "kernel": KernelProvider(), + "browserbase": BrowserbaseProvider(), + "steel": SteelProvider(), + "hyperbrowser": HyperbrowserProvider(), + "anchor": AnchorProvider(), +} + + +def get(name: str) -> Provider: + if name not in PROVIDERS: + raise SystemExit(f"Unknown browser provider {name!r}; known: {', '.join(PROVIDERS)}") + return PROVIDERS[name] diff --git a/harness/run.py b/harness/run.py index f18b53f..a1aa941 100644 --- a/harness/run.py +++ b/harness/run.py @@ -11,9 +11,16 @@ from pathlib import Path from typing import Any -from harness import preflight +from harness import preflight, providers from harness.configure import configure_tasks -from harness.definitions import Arm, load_arm, load_benchmark, load_suite, load_toolset +from harness.definitions import ( + ENVIRONMENTS, + Arm, + load_arm, + load_benchmark, + load_suite, + load_toolset, +) from harness.environment import RUN_ID_ENV, delete_run_proxies # Transient failures worth retrying as a whole trial: Kernel API hiccups and @@ -104,6 +111,10 @@ def run( prepared = prepare( benchmark_name, arm_names, task_ids, work_dir, toolset_ref, suite_args, toolset_args ) + for item in prepared: + if item.arm.environment == "remote-browser": + for configuration in item.configurations.values(): + preflight.require_env(*providers.get(configuration["browser"]["provider"]).env) jobs_dir.mkdir(parents=True, exist_ok=True) manifest_path = jobs_dir / f"run-{run_id}.json" @@ -146,7 +157,7 @@ def harbor_command( "--path", str(item.dataset), "--agent", arm.agent, "--model", arm.model, - "--env", "harness.environment:KernelTaskEnvironment", + "--env", ENVIRONMENTS[arm.environment], "--job-name", job_name, "--jobs-dir", str(jobs_dir), "--n-concurrent", str(concurrency), diff --git a/suites/retail-cart/image/Dockerfile b/suites/retail-cart/image/Dockerfile index 4362d77..78620f0 100644 --- a/suites/retail-cart/image/Dockerfile +++ b/suites/retail-cart/image/Dockerfile @@ -1,13 +1,18 @@ # Task image for retail-cart. The browser is the Kernel VM this container runs -# in, so the image only holds the finalize tool, the start-URL setup and the -# scorer. Toolsets append their own layers. +# in, or a remote browser reached over CDP, so the image only holds the +# finalize tool, the start-URL setup and the scorer. Toolsets append their own +# layers. FROM node:22-bookworm-slim RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates curl git python3 \ && rm -rf /var/lib/apt/lists/* +# cdp-execute drives a remote browser; it needs no browser of its own. +RUN npm install --global playwright-core@1.64.1 +ENV NODE_PATH=/usr/local/lib/node_modules + COPY bench/ /opt/bench/ -RUN chmod +x /opt/bench/finalize-mcp /opt/bench/open-start-url /opt/bench/score +RUN chmod +x /opt/bench/cdp-execute /opt/bench/finalize-mcp /opt/bench/open-start-url /opt/bench/score WORKDIR /app diff --git a/suites/retail-cart/image/bench/cdp-execute b/suites/retail-cart/image/bench/cdp-execute new file mode 100755 index 0000000..17010ba --- /dev/null +++ b/suites/retail-cart/image/bench/cdp-execute @@ -0,0 +1,34 @@ +#!/usr/bin/env node +// Run Playwright code against the trial's remote browser over CDP, the way +// Kernel's playwright/execute API runs it: the code on stdin is the body of an +// async function given `page`, `context` and `browser`. Prints +// {success, result} or {success: false, error} as JSON. +const fs = require("fs"); +const { chromium } = require("playwright-core"); + +const CDP_URL_FILE = "/run/browser/cdp-url"; + +async function main() { + const code = fs.readFileSync(0, "utf8"); + const timeoutMs = Number(process.argv[2] || 60) * 1000; + const timer = setTimeout(() => { + console.log(JSON.stringify({ success: false, error: `timed out after ${timeoutMs} ms` })); + process.exit(0); + }, timeoutMs); + const browser = await chromium.connectOverCDP(fs.readFileSync(CDP_URL_FILE, "utf8").trim()); + const context = browser.contexts()[0] ?? (await browser.newContext()); + // The page the agent was last working in: the newest one that is not blank. + const pages = context.pages().filter((p) => p.url() !== "about:blank"); + const page = pages.at(-1) ?? context.pages()[0] ?? (await context.newPage()); + const run = new (Object.getPrototypeOf(async () => {}).constructor)("page", "context", "browser", code); + const result = await run(page, context, browser); + clearTimeout(timer); + console.log(JSON.stringify({ success: true, result: result ?? null })); + // Exit without closing the browser: the session belongs to the trial. + process.exit(0); +} + +main().catch((error) => { + console.log(JSON.stringify({ success: false, error: String(error && error.message ? error.message : error) })); + process.exit(0); +}); diff --git a/suites/retail-cart/image/bench/finalize-mcp b/suites/retail-cart/image/bench/finalize-mcp index 9339d7a..3d07f23 100755 --- a/suites/retail-cart/image/bench/finalize-mcp +++ b/suites/retail-cart/image/bench/finalize-mcp @@ -35,8 +35,8 @@ OUTCOMES = { "to load." ), "kernel_error": ( - "the Kernel browser or its tools kept you from finishing: the browser REPL or the " - "browser itself errored, hung, or crashed." + "the browser or its tools kept you from finishing: the browser tools or the browser " + "itself errored, hung, or crashed." ), } diff --git a/suites/retail-cart/image/bench/open-start-url b/suites/retail-cart/image/bench/open-start-url index d6dd7f3..3ee95f5 100755 --- a/suites/retail-cart/image/bench/open-start-url +++ b/suites/retail-cart/image/bench/open-start-url @@ -9,10 +9,15 @@ from __future__ import annotations import json import os +import subprocess import sys import time import urllib.error import urllib.request +from pathlib import Path + +# Written by the remote-browser environment; absent in a Kernel browser's VM. +CDP_URL_FILE = Path("/run/browser/cdp-url") CODE = """ await page.goto(%s, { waitUntil: "domcontentloaded", timeout: 45000 }); @@ -22,10 +27,29 @@ return page.url(); def main(url: str) -> int: + if CDP_URL_FILE.exists(): + output = subprocess.run( + ["/opt/bench/cdp-execute", "90"], + input=CODE % json.dumps(url), + capture_output=True, + text=True, + timeout=120, + ) + result = json.loads(output.stdout) + else: + result = execute_with_kernel(CODE % json.dumps(url)) + if not result.get("success"): + print(f"Could not open {url}: {result.get('error')}", file=sys.stderr) + return 1 + print(f"Opened {result.get('result')}") + return 0 + + +def execute_with_kernel(code: str) -> dict: base = (os.environ.get("KERNEL_BASE_URL") or "https://api.onkernel.com").rstrip("/") request = urllib.request.Request( f"{base}/browsers/{os.environ['KERNEL_SESSION_ID']}/playwright/execute", - data=json.dumps({"code": CODE % json.dumps(url), "timeout_sec": 90}).encode(), + data=json.dumps({"code": code, "timeout_sec": 90}).encode(), headers={ "Authorization": f"Bearer {os.environ['KERNEL_API_KEY']}", "Content-Type": "application/json", @@ -44,11 +68,7 @@ def main(url: str) -> int: if attempt == 4: raise time.sleep(2 * (attempt + 1)) - if not result.get("success"): - print(f"Could not open {url}: {result.get('error')}", file=sys.stderr) - return 1 - print(f"Opened {result.get('result')}") - return 0 + return result if __name__ == "__main__": diff --git a/suites/retail-cart/image/bench/score b/suites/retail-cart/image/bench/score index 175d4a8..4bcff6e 100644 --- a/suites/retail-cart/image/bench/score +++ b/suites/retail-cart/image/bench/score @@ -7,7 +7,9 @@ finalize, and `outcome_` which failure it reported, so bot blocks can be told apart from other website errors, Kernel errors and the agent's own failures. -The browser's final URL and title are saved, then the verifier checks the +The browser is the Kernel VM's, reached through Kernel's API, or a remote +browser reached over CDP when the environment wrote its URL. The browser's +final URL and title are saved, then the verifier checks the cart itself: it reads the page the agent ended on when that is already the cart, otherwise opens the task's `cart_url`, or fetches its `cart_api` from the page, and sets `cart_verified` when the text matches `cart_match`. When @@ -22,12 +24,15 @@ import json import os import re import shutil +import subprocess import urllib.parse import urllib.request from pathlib import Path REPORT = Path("/logs/agent/finalize.json") TASK = Path("/opt/bench/task.json") +# Written by the remote-browser environment; absent in a Kernel browser's VM. +CDP_URL_FILE = Path("/run/browser/cdp-url") VERIFIER = Path("/logs/verifier") FAILURES = ("failed", "blocked", "website_error", "kernel_error") @@ -69,6 +74,8 @@ return await page.evaluate(async (url) => { def execute(code: str, timeout_sec: int) -> dict: + if CDP_URL_FILE.exists(): + return execute_over_cdp(code, timeout_sec) base = (os.environ.get("KERNEL_BASE_URL") or "https://api.onkernel.com").rstrip("/") request = urllib.request.Request( f"{base}/browsers/{os.environ['KERNEL_SESSION_ID']}/playwright/execute", @@ -89,6 +96,23 @@ def execute(code: str, timeout_sec: int) -> dict: return result.get("result") or {} +def execute_over_cdp(code: str, timeout_sec: int) -> dict: + try: + output = subprocess.run( + ["/opt/bench/cdp-execute", str(timeout_sec)], + input=code, + capture_output=True, + text=True, + timeout=timeout_sec + 30, + ) + result = json.loads(output.stdout) + except (OSError, ValueError, subprocess.SubprocessError) as error: + return {"error": str(error)} + if not result.get("success"): + return {"error": result.get("error")} + return result.get("result") or {} + + def same_page(url: str, other: str) -> bool: a, b = urllib.parse.urlsplit(url), urllib.parse.urlsplit(other) return (a.netloc, a.path.rstrip("/").lower()) == (b.netloc, b.path.rstrip("/").lower()) diff --git a/tests/test_definitions.py b/tests/test_definitions.py index 2ca3d28..30fba7e 100644 --- a/tests/test_definitions.py +++ b/tests/test_definitions.py @@ -1,3 +1,6 @@ +import pytest + +from harness import definitions, providers from harness.definitions import ROOT, load_arm, load_benchmark @@ -10,3 +13,22 @@ def test_every_benchmark_and_arm_resolves(): assert (ROOT / "toolsets" / arm.toolset / "toolset.py").is_file() if arm.browser.configurator: assert (ROOT / "configurators" / arm.browser.configurator / "configure").is_file() + + +def test_provider_arms_run_on_remote_browsers(): + benchmark = load_benchmark("retail-cart-providers") + for name in benchmark.arms: + arm = load_arm(name) + assert arm.environment == "remote-browser" + assert arm.toolset == "playwright-mcp" + assert arm.browser.config["provider"] in providers.PROVIDERS + + +def test_unknown_environment_is_refused(tmp_path, monkeypatch): + (tmp_path / "arms").mkdir() + (tmp_path / "arms/bad.toml").write_text( + 'agent = "codex"\nmodel = "m"\ntoolset = "t"\nenvironment = "elsewhere"\n' + ) + monkeypatch.setattr(definitions, "ROOT", tmp_path) + with pytest.raises(SystemExit, match="elsewhere"): + load_arm("bad") diff --git a/tests/test_environment.py b/tests/test_environment.py index 2f282eb..0f0b523 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -1,5 +1,6 @@ import asyncio import json +import logging import pytest from harbor.environments.kernel import KernelEnvironment @@ -195,3 +196,74 @@ def test_vm_setup_commands_run_as_root_after_the_browser_is_created(tmp_path, mo asyncio.run(env._create_browser()) assert calls == [("exec", ["-c", "echo hi"], True), ("exec", ["-c", "true"], True)] + + +class FakeProvider: + env = () + + def __init__(self, fail=False): + self.fail = fail + self.calls = [] + + async def create(self, params): + self.calls.append(("create", params)) + if self.fail: + raise RuntimeError("no capacity") + return environment.providers.Session("remote-1", "wss://remote/1") + + async def release(self, session_id): + self.calls.append(("release", session_id)) + + +def _remote_environment(tmp_path, monkeypatch, provider): + from harbor.environments.base import ExecResult + from harbor.environments.docker.docker import DockerEnvironment + + calls = [] + + async def start(self, force_build): + calls.append("container started") + + async def stop(self, delete): + calls.append("container stopped") + + async def exec(self, command, env=None, user=None, **kwargs): + calls.append((command, env, user)) + return ExecResult(return_code=0) + + def init(self, environment_dir, **kwargs): + self._persistent_env = {} + self.logger = logging.getLogger("test") + + monkeypatch.setattr(DockerEnvironment, "__init__", init) + monkeypatch.setattr(DockerEnvironment, "start", start) + monkeypatch.setattr(DockerEnvironment, "stop", stop) + monkeypatch.setattr(DockerEnvironment, "exec", exec) + monkeypatch.setitem(environment.providers.PROVIDERS, "fake", provider) + (tmp_path / "kernel-config.json").write_text( + json.dumps({"provider": "fake", "params": {"stealth": True}}) + ) + return environment.RemoteBrowserEnvironment(environment_dir=tmp_path), calls + + +def test_remote_browser_is_created_after_the_container_and_released_after(tmp_path, monkeypatch): + provider = FakeProvider() + env, calls = _remote_environment(tmp_path, monkeypatch, provider) + + asyncio.run(env.start(force_build=False)) + asyncio.run(env.stop(delete=True)) + + assert calls[0] == "container started" + command, exec_env, user = calls[1] + assert environment.CDP_URL_FILE in command + assert exec_env == {"CDP_URL": "wss://remote/1"} + assert user == "root" + assert env._persistent_env["BROWSER_SESSION_ID"] == "remote-1" + assert calls[2] == "container stopped" + assert provider.calls == [("create", {"stealth": True}), ("release", "remote-1")] + + +def test_remote_browser_creation_failure_is_identifiable(tmp_path, monkeypatch): + env, _ = _remote_environment(tmp_path, monkeypatch, FakeProvider(fail=True)) + with pytest.raises(environment.BrowserCreationError, match="no capacity"): + asyncio.run(env.start(force_build=False)) diff --git a/tests/test_playwright_mcp.py b/tests/test_playwright_mcp.py new file mode 100644 index 0000000..35e8aaa --- /dev/null +++ b/tests/test_playwright_mcp.py @@ -0,0 +1,20 @@ +import tomllib + +from harness.definitions import load_toolset + + +def test_apply_adds_the_server_install_and_instruction(task_dir): + load_toolset("playwright-mcp").apply([task_dir]) + + environment = task_dir / "environment" + assert (environment / "playwright-mcp/bin/browser-mcp").is_file() + assert "npm install --global @playwright/mcp@0.0.84" in (environment / "Dockerfile").read_text() + config = tomllib.loads((task_dir / "task.toml").read_text()) + assert config["environment"]["mcp_servers"][-1]["command"] == "/usr/local/bin/browser-mcp" + assert config["metadata"]["playwright_mcp_version"] == "0.0.84" + assert "`browser` MCP server" in (task_dir / "steps/run/instruction.md").read_text() + + +def test_ref_picks_the_playwright_mcp_version(task_dir): + load_toolset("playwright-mcp").apply([task_dir], ref="0.0.80") + assert "@playwright/mcp@0.0.80" in (task_dir / "environment/Dockerfile").read_text() diff --git a/tests/test_providers.py b/tests/test_providers.py new file mode 100644 index 0000000..9a9cbdf --- /dev/null +++ b/tests/test_providers.py @@ -0,0 +1,86 @@ +import asyncio +import json + +import httpx +import pytest + +from harness import providers + + +def _serve(monkeypatch, provider, handler): + requests = [] + + def record(request): + requests.append(request) + return handler(request) + + monkeypatch.setattr( + provider, + "_client", + lambda: httpx.AsyncClient( + base_url=provider.base_url, + headers={provider.key_header: "key"}, + transport=httpx.MockTransport(record), + ), + ) + return requests + + +def test_browserbase_sends_the_project_and_releases_the_session(monkeypatch): + monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "proj") + provider = providers.get("browserbase") + requests = _serve( + monkeypatch, + provider, + lambda request: httpx.Response(200, json={"id": "bb1", "connectUrl": "wss://bb/1"}), + ) + + session = asyncio.run(provider.create({"keepAlive": True})) + asyncio.run(provider.release(session.id)) + + assert session == providers.Session("bb1", "wss://bb/1") + assert json.loads(requests[0].content) == {"projectId": "proj", "keepAlive": True} + assert requests[1].url.path == "/v1/sessions/bb1" + assert json.loads(requests[1].content)["status"] == "REQUEST_RELEASE" + + +def test_steel_connects_with_the_session_and_key(monkeypatch): + monkeypatch.setenv("STEEL_API_KEY", "sk") + provider = providers.get("steel") + requests = _serve(monkeypatch, provider, lambda request: httpx.Response(200, json={"id": "s1"})) + + session = asyncio.run(provider.create({"useProxy": True})) + asyncio.run(provider.release("s1")) + + assert session.cdp_url == "wss://connect.steel.dev?apiKey=sk&sessionId=s1" + assert [request.url.path for request in requests] == ["/v1/sessions", "/v1/sessions/s1/release"] + + +def test_hyperbrowser_and_anchor_read_their_cdp_urls(monkeypatch): + hyperbrowser = providers.get("hyperbrowser") + _serve( + monkeypatch, + hyperbrowser, + lambda request: httpx.Response(200, json={"id": "h1", "wsEndpoint": "wss://hb/1"}), + ) + anchor = providers.get("anchor") + _serve( + monkeypatch, + anchor, + lambda request: httpx.Response(200, json={"data": {"id": "a1", "cdp_url": "wss://a/1"}}), + ) + + assert asyncio.run(hyperbrowser.create({})) == providers.Session("h1", "wss://hb/1") + assert asyncio.run(anchor.create({})) == providers.Session("a1", "wss://a/1") + + +def test_failed_create_raises(monkeypatch): + provider = providers.get("hyperbrowser") + _serve(monkeypatch, provider, lambda request: httpx.Response(402, json={})) + with pytest.raises(httpx.HTTPStatusError): + asyncio.run(provider.create({})) + + +def test_unknown_provider_is_refused(): + with pytest.raises(SystemExit, match="nope"): + providers.get("nope") diff --git a/tests/test_run.py b/tests/test_run.py index e74d2bb..e91909a 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -67,3 +67,9 @@ def test_harbor_command_passes_agent_kwargs_as_json(tmp_path): assert "version=0.163.0" in command assert "disable_web_search=true" in command assert command[command.index("--n-attempts") + 1] == "3" + + +def test_harbor_command_picks_the_arms_environment(tmp_path): + arm = Arm(name="a", agent="codex", model="m", toolset="t", environment="remote-browser") + command = run.harbor_command(run.PreparedArm(arm, tmp_path, {}), tmp_path, "job", 1) + assert command[command.index("--env") + 1] == "harness.environment:RemoteBrowserEnvironment" diff --git a/toolsets/playwright-mcp/image/bin/browser-mcp b/toolsets/playwright-mcp/image/bin/browser-mcp new file mode 100755 index 0000000..8063fa4 --- /dev/null +++ b/toolsets/playwright-mcp/image/bin/browser-mcp @@ -0,0 +1,6 @@ +#!/bin/sh +# Playwright MCP attached over CDP to the trial's remote browser, whose URL the +# environment writes to a file before the agent starts. +set -eu +exec playwright-mcp --cdp-endpoint "$(cat /run/browser/cdp-url)" \ + --cdp-timeout 60000 --output-dir /tmp/playwright-mcp "$@" diff --git a/toolsets/playwright-mcp/image/layer.Dockerfile b/toolsets/playwright-mcp/image/layer.Dockerfile new file mode 100644 index 0000000..d3753c1 --- /dev/null +++ b/toolsets/playwright-mcp/image/layer.Dockerfile @@ -0,0 +1,3 @@ +# playwright-mcp toolset. Appended to the suite's Dockerfile, which must have +# Node.js. The toolset appends the line installing its @playwright/mcp version. +COPY playwright-mcp/bin/ /usr/local/bin/ diff --git a/toolsets/playwright-mcp/toolset.py b/toolsets/playwright-mcp/toolset.py new file mode 100644 index 0000000..c6639fe --- /dev/null +++ b/toolsets/playwright-mcp/toolset.py @@ -0,0 +1,61 @@ +"""Playwright MCP, attached over CDP to a browser from any provider. + +The same tools drive every provider's browser, so arms that differ only in +provider compare the browsers rather than the tools. Needs an arm with +`environment = "remote-browser"`, which writes the browser's CDP URL into the +task container. The agent works in the browser's existing page and context. + +Toolset ref: the @playwright/mcp version to install, default `VERSION`. +""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import tomlkit + +from harness.tasks import append_instruction, read_config, write_config + +VERSION = "0.0.84" +IMAGE = Path(__file__).with_name("image") + +INSTRUCTION = """ +## Browser tools + +- Use the `browser` MCP server's tools for all browser interaction. They drive a browser that is already open on the start page; keep working in it. +- Do not install or launch another browser. +""" + + +def apply(task_dirs: list[Path], ref: str | None = None) -> None: + for task_dir in task_dirs: + _apply(task_dir, ref or VERSION) + + +def _apply(task_dir: Path, version: str) -> None: + context = task_dir / "environment" / "playwright-mcp" + shutil.copytree(IMAGE / "bin", context / "bin") + dockerfile = task_dir / "environment" / "Dockerfile" + dockerfile.write_text( + dockerfile.read_text().rstrip() + + "\n\n" + + (IMAGE / "layer.Dockerfile").read_text() + + f"RUN npm install --global @playwright/mcp@{version}\n" + ) + + config = read_config(task_dir) + server = tomlkit.table() + server.update( + { + "name": "browser", + "transport": "stdio", + "command": "/usr/local/bin/browser-mcp", + "args": [], + } + ) + servers = config["environment"].setdefault("mcp_servers", tomlkit.aot()) + servers.append(server) + config.setdefault("metadata", tomlkit.table())["playwright_mcp_version"] = version + write_config(task_dir, config) + append_instruction(task_dir, INSTRUCTION) From 55d0d775e6752d569fcb24945f254efba173d5ac Mon Sep 17 00:00:00 2001 From: Andrew Steele Date: Mon, 9 Nov 2026 16:46:29 -0500 Subject: [PATCH 2/2] Run retail-cart-providers with Claude on six providers, and grow retail-cart to 28 tiered sites Providers: - Hyperbrowser: add keepAlive=true to the CDP URL. Without it the session stops when the start page disconnects, so every trial lost its browser. - Browser Use: new provider (API v4 browsers, stopped with PATCH). - Steel: send STEEL_PROJECT_ID for organization-scoped keys; Steel arms use 15-minute sessions, the Launch plan's maximum. - Browserbase: drop Advanced Stealth, which Browserbase rejects outside the Enterprise plan ("Verified mode"). - Kernel: send create params the SDK does not know yet (the closed-beta macOS fingerprint) in the request body. Environment: kernel-vm-remote-browser runs the task container on a Kernel VM used only as compute and drives the provider's browser over CDP, so provider arms run without local Docker. Kernel arms still get a separate browser. Arms: claude-pw-{kernel,kernel-macos,browserbase,browser-use,hyperbrowser, steel} on that environment, codex-pw-browser-use, and claude-repl for retail-cart. retail-cart: 28 sites in easy (2), standard (13) and hard (13) tiers, with --suite-arg tiers=... to run one. The verifier no longer counts an empty cart as verified when its recommendations mention the item. The pinned configurator runs sites it has no measurements for on the arm's own config. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/_run.yml | 2 + README.md | 26 ++--- arms/claude-pw-browser-use.toml | 16 +++ arms/claude-pw-browserbase.toml | 17 ++++ arms/claude-pw-hyperbrowser.toml | 16 +++ arms/claude-pw-kernel-macos.toml | 17 ++++ arms/claude-pw-kernel.toml | 16 +++ arms/claude-pw-steel.toml | 17 ++++ arms/claude-repl.toml | 16 +++ arms/codex-pw-browser-use.toml | 14 +++ arms/codex-pw-browserbase.toml | 7 +- arms/codex-pw-steel.toml | 4 +- benchmarks/retail-cart-providers.toml | 7 ++ benchmarks/retail-cart.toml | 5 +- configurators/pinned/configure | 7 +- harness/definitions.py | 4 +- harness/environment.py | 68 ++++++++++--- harness/providers.py | 46 ++++++++- harness/run.py | 2 +- suites/retail-cart/image/bench/score | 11 +- suites/retail-cart/suite.py | 25 ++++- suites/retail-cart/tasks.toml | 139 ++++++++++++++++++++++++-- tests/test_definitions.py | 2 +- tests/test_environment.py | 32 ++++-- tests/test_pinned.py | 13 ++- tests/test_providers.py | 61 ++++++++++- tests/test_retail_cart.py | 13 +++ 27 files changed, 540 insertions(+), 63 deletions(-) create mode 100644 arms/claude-pw-browser-use.toml create mode 100644 arms/claude-pw-browserbase.toml create mode 100644 arms/claude-pw-hyperbrowser.toml create mode 100644 arms/claude-pw-kernel-macos.toml create mode 100644 arms/claude-pw-kernel.toml create mode 100644 arms/claude-pw-steel.toml create mode 100644 arms/claude-repl.toml create mode 100644 arms/codex-pw-browser-use.toml diff --git a/.github/workflows/_run.yml b/.github/workflows/_run.yml index 106f2ea..701c8be 100644 --- a/.github/workflows/_run.yml +++ b/.github/workflows/_run.yml @@ -69,6 +69,8 @@ jobs: BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} STEEL_API_KEY: ${{ secrets.STEEL_API_KEY }} + STEEL_PROJECT_ID: ${{ secrets.STEEL_PROJECT_ID }} + BROWSER_USE_API_KEY: ${{ secrets.BROWSER_USE_API_KEY }} HYPERBROWSER_API_KEY: ${{ secrets.HYPERBROWSER_API_KEY }} ANCHOR_API_KEY: ${{ secrets.ANCHOR_API_KEY }} BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} diff --git a/README.md b/README.md index 10bb302..efefce0 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ A run of two arms on two tasks (Red Cross and Indeed) produced this summary: | `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` | Codex and Claude Code arms; the `site-config` configurator's model | | `PURELY_MAIL_API_KEY`, `PURELY_MAIL_DOMAIN` | ClawBench creates a disposable [PurelyMail](https://purelymail.com) mailbox on your domain for every task, so agents can sign up and receive confirmation emails. | | `CLAWBENCH_JUDGE_BASE_URL`, `CLAWBENCH_JUDGE_API_KEY`, `CLAWBENCH_JUDGE_MODEL`, `CLAWBENCH_JUDGE_API_TYPE` | ClawBench's LLM judge, which checks the intercepted request against the task. Any OpenAI-compatible endpoint works with `CLAWBENCH_JUDGE_API_TYPE=openai-completions`; we use `deepseek/deepseek-v4-pro` through OpenRouter. See [ClawBench](https://github.com/TIGER-AI-Lab/ClawBench) for details. | -| `BROWSERBASE_API_KEY`, `BROWSERBASE_PROJECT_ID`, `STEEL_API_KEY`, `HYPERBROWSER_API_KEY`, `ANCHOR_API_KEY` | Arms on those providers' browsers (`environment = "remote-browser"`); checked before a run that includes them | +| `BROWSERBASE_API_KEY`, `BROWSERBASE_PROJECT_ID`, `BROWSER_USE_API_KEY`, `STEEL_API_KEY` (plus `STEEL_PROJECT_ID` for an organization-scoped key), `HYPERBROWSER_API_KEY`, `ANCHOR_API_KEY` | Arms on those providers' browsers (`environment = "remote-browser"` or `"kernel-vm-remote-browser"`); checked before a run that includes them | | `BRAINTRUST_API_KEY`, `BRAINTRUST_PROJECT` | Optional: publishing results | ## Running in GitHub Actions @@ -122,7 +122,7 @@ ClawBench's primary score is the lenient reward. `kernel_mcp_valid` records whet ### `retail-cart` -Guest add-to-cart tasks on 16 retail and industrial-supply sites with heavy bot protection. Each task starts on the site's home page and asks the agent to put one item in the site's cart as a guest, by whatever route works, and show the cart. +Guest add-to-cart tasks on 28 retail and industrial-supply sites, in three tiers: 2 `easy` controls with little bot protection, 13 `standard` protected sites, and 13 `hard` sites that block most automated browsers. Run one tier with `--suite-arg tiers=hard`. Each task starts on the site's home page and asks the agent to put one item in the site's cart as a guest, by whatever route works, and show the cart. | Arm | Agent | Browser | |---|---|---| @@ -142,17 +142,19 @@ uv run bench run retail-cart --attempts 10 ### `retail-cart-providers` -The `retail-cart` tasks on each browser provider's browser. Every arm uses the same agent (Codex, `gpt-6-luna`), the same tools ([Playwright MCP](https://github.com/microsoft/playwright-mcp) attached over CDP) and the same task container on the runner's Docker, so arms differ only in the browser. Each provider gets its own anti-bot settings: +The `retail-cart` tasks on each browser provider's browser. Within an agent, every arm uses the same tools ([Playwright MCP](https://github.com/microsoft/playwright-mcp) attached over CDP) and the same task container, so arms differ only in the browser. The `codex-pw-*` arms (Codex, `gpt-6-luna`) run the container on the runner's Docker (`environment = "remote-browser"`). The `claude-pw-*` arms (Claude Code, `claude-sonnet-5-5`) run it on a Kernel VM used only as compute (`environment = "kernel-vm-remote-browser"`), which needs no local Docker; the Kernel arms still get a separate browser session, reached over the network like every other provider's. Each provider gets its own anti-bot settings: -| Arm | Browser | -|---|---| -| `codex-pw-kernel` | Kernel, `stealth: true` | -| `codex-pw-browserbase` | Browserbase, Advanced Stealth (Scale plan), CAPTCHA solving, its residential proxies | -| `codex-pw-steel` | Steel, residential proxy and CAPTCHA solving | -| `codex-pw-hyperbrowser` | Hyperbrowser, stealth, proxy and CAPTCHA solving | -| `codex-pw-anchor` | Anchor, its US proxy, extra stealth and CAPTCHA solving | - -The start page, the agent and the verifier each connect to the browser separately, so sessions are created to outlive a CDP disconnect (Browserbase's `keepAlive`, Anchor's idle timeout). Scoring is the same as `retail-cart`. +| Provider | Arms | Browser | +|---|---|---| +| Kernel | `codex-pw-kernel`, `claude-pw-kernel` | `stealth: true` (managed ISP proxy and CAPTCHA solver) | +| Kernel, macOS | `claude-pw-kernel-macos` | as above, plus the macOS fingerprint (`fingerprint: {os: macos}`, closed beta) | +| Browserbase | `codex-pw-browserbase`, `claude-pw-browserbase` | CAPTCHA solving and its residential proxies. Advanced Stealth is left out: Browserbase rejects it outside the Enterprise plan | +| Browser Use | `codex-pw-browser-use`, `claude-pw-browser-use` | its US proxy | +| Steel | `codex-pw-steel`, `claude-pw-steel` | residential proxy and CAPTCHA solving; 15-minute sessions, the Launch plan's maximum | +| Hyperbrowser | `codex-pw-hyperbrowser`, `claude-pw-hyperbrowser` | stealth, proxy and CAPTCHA solving | +| Anchor | `codex-pw-anchor` | its US proxy, extra stealth and CAPTCHA solving | + +The start page, the agent and the verifier each connect to the browser separately, so sessions are created to outlive a CDP disconnect (Browserbase's `keepAlive`, Hyperbrowser's `keepAlive=true` on the CDP URL, Anchor's idle timeout). A Steel key scoped to the whole organization also needs `STEEL_PROJECT_ID`. Scoring is the same as `retail-cart`. ```bash uv run bench run retail-cart-providers --attempts 10 --concurrency 4 diff --git a/arms/claude-pw-browser-use.toml b/arms/claude-pw-browser-use.toml new file mode 100644 index 0000000..47b4b36 --- /dev/null +++ b/arms/claude-pw-browser-use.toml @@ -0,0 +1,16 @@ +# Claude Code driving a browser-use browser through Playwright MCP over CDP, the +# same agent, tools and task container as every claude-pw-* arm. The container +# runs on a Kernel VM used only as compute; the browser is a separate session. +# Browser: Browser Use's cloud browser with its managed US proxy. +agent = "claude-code" +model = "claude-sonnet-5-5" +agent_version = "2.1.238" +toolset = "playwright-mcp" +environment = "kernel-vm-remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "browser-use" +params = { proxyCountryCode = "us", timeout = 30 } diff --git a/arms/claude-pw-browserbase.toml b/arms/claude-pw-browserbase.toml new file mode 100644 index 0000000..c5e79e4 --- /dev/null +++ b/arms/claude-pw-browserbase.toml @@ -0,0 +1,17 @@ +# Claude Code driving a browserbase browser through Playwright MCP over CDP, the +# same agent, tools and task container as every claude-pw-* arm. The container +# runs on a Kernel VM used only as compute; the browser is a separate session. +# Browser: Browserbase's managed proxies and CAPTCHA solving. Advanced stealth is left +# out: Browserbase rejects it outside the Enterprise plan ("Verified mode"). +agent = "claude-code" +model = "claude-sonnet-5-5" +agent_version = "2.1.238" +toolset = "playwright-mcp" +environment = "kernel-vm-remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "browserbase" +params = { keepAlive = true, timeout = 1800, region = "us-east-1", proxies = true, browserSettings = { solveCaptchas = true } } diff --git a/arms/claude-pw-hyperbrowser.toml b/arms/claude-pw-hyperbrowser.toml new file mode 100644 index 0000000..c059310 --- /dev/null +++ b/arms/claude-pw-hyperbrowser.toml @@ -0,0 +1,16 @@ +# Claude Code driving a hyperbrowser browser through Playwright MCP over CDP, the +# same agent, tools and task container as every claude-pw-* arm. The container +# runs on a Kernel VM used only as compute; the browser is a separate session. +# Browser: Hyperbrowser's stealth, managed US proxy and CAPTCHA solving. +agent = "claude-code" +model = "claude-sonnet-5-5" +agent_version = "2.1.238" +toolset = "playwright-mcp" +environment = "kernel-vm-remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "hyperbrowser" +params = { useStealth = true, useProxy = true, proxyCountry = "US", solveCaptchas = true, timeoutMinutes = 30 } diff --git a/arms/claude-pw-kernel-macos.toml b/arms/claude-pw-kernel-macos.toml new file mode 100644 index 0000000..67860bc --- /dev/null +++ b/arms/claude-pw-kernel-macos.toml @@ -0,0 +1,17 @@ +# Claude Code driving a kernel browser through Playwright MCP over CDP, the +# same agent, tools and task container as every claude-pw-* arm. The container +# runs on a Kernel VM used only as compute; the browser is a separate session. +# Browser: Kernel's anti-detection mode plus its macOS fingerprint (closed beta): macOS +# user agent and platform, Apple GPU renderer. +agent = "claude-code" +model = "claude-sonnet-5-5" +agent_version = "2.1.238" +toolset = "playwright-mcp" +environment = "kernel-vm-remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "kernel" +params = { stealth = true, timeout_seconds = 1800, fingerprint = { os = "macos" } } diff --git a/arms/claude-pw-kernel.toml b/arms/claude-pw-kernel.toml new file mode 100644 index 0000000..8ebff5f --- /dev/null +++ b/arms/claude-pw-kernel.toml @@ -0,0 +1,16 @@ +# Claude Code driving a kernel browser through Playwright MCP over CDP, the +# same agent, tools and task container as every claude-pw-* arm. The container +# runs on a Kernel VM used only as compute; the browser is a separate session. +# Browser: Kernel's anti-detection mode (`stealth`): managed ISP proxy and CAPTCHA solver. +agent = "claude-code" +model = "claude-sonnet-5-5" +agent_version = "2.1.238" +toolset = "playwright-mcp" +environment = "kernel-vm-remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "kernel" +params = { stealth = true, timeout_seconds = 1800 } diff --git a/arms/claude-pw-steel.toml b/arms/claude-pw-steel.toml new file mode 100644 index 0000000..f07016a --- /dev/null +++ b/arms/claude-pw-steel.toml @@ -0,0 +1,17 @@ +# Claude Code driving a steel browser through Playwright MCP over CDP, the +# same agent, tools and task container as every claude-pw-* arm. The container +# runs on a Kernel VM used only as compute; the browser is a separate session. +# Browser: Steel's managed residential proxy and CAPTCHA solving. 15-minute sessions: the +# Launch plan's maximum (other arms get 30), so a long trial can lose its browser. +agent = "claude-code" +model = "claude-sonnet-5-5" +agent_version = "2.1.238" +toolset = "playwright-mcp" +environment = "kernel-vm-remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "steel" +params = { useProxy = true, solveCaptcha = true, timeout = 900000 } diff --git a/arms/claude-repl.toml b/arms/claude-repl.toml new file mode 100644 index 0000000..e07e233 --- /dev/null +++ b/arms/claude-repl.toml @@ -0,0 +1,16 @@ +# Same as codex-repl, but with Claude Code driving the browser only through +# Kernel MCP's browser REPL, with a fixed stealth browser. +agent = "claude-code" +model = "claude-sonnet-5-5" +agent_version = "2.1.238" +toolset = "kernel-mcp-local" + +# Claude Code's own web tools would let the agent answer without the browser. +[agent_kwargs] +disable_web_search = true + +[toolset_args] +toolsets = ["repl"] + +[browser] +config = { stealth = true } diff --git a/arms/codex-pw-browser-use.toml b/arms/codex-pw-browser-use.toml new file mode 100644 index 0000000..659dd1c --- /dev/null +++ b/arms/codex-pw-browser-use.toml @@ -0,0 +1,14 @@ +# Codex driving a Browser Use cloud browser through Playwright MCP over CDP, with +# Browser Use's managed US proxy. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "browser-use" +params = { proxyCountryCode = "us", timeout = 45 } diff --git a/arms/codex-pw-browserbase.toml b/arms/codex-pw-browserbase.toml index 18f379a..ef045a0 100644 --- a/arms/codex-pw-browserbase.toml +++ b/arms/codex-pw-browserbase.toml @@ -1,6 +1,7 @@ # Codex driving a Browserbase browser through Playwright MCP over CDP, with -# Browserbase's anti-bot settings: Advanced Stealth (Scale plan), CAPTCHA -# solving and its residential proxies. keepAlive keeps the session across the +# Browserbase's anti-bot settings: CAPTCHA solving and its residential proxies. +# Advanced Stealth is left out: Browserbase rejects it outside the Enterprise +# plan ("Verified mode is only available on the Enterprise plan"). keepAlive keeps the session across the # start page, agent and verifier connections. agent = "codex" model = "gpt-6-luna" @@ -18,4 +19,4 @@ provider = "browserbase" keepAlive = true timeout = 2700 proxies = true -browserSettings = { advancedStealth = true, solveCaptchas = true } +browserSettings = { solveCaptchas = true } diff --git a/arms/codex-pw-steel.toml b/arms/codex-pw-steel.toml index b78b0e6..0550218 100644 --- a/arms/codex-pw-steel.toml +++ b/arms/codex-pw-steel.toml @@ -11,5 +11,5 @@ disable_web_search = true [browser.config] provider = "steel" -# timeout is in milliseconds. -params = { useProxy = true, solveCaptcha = true, timeout = 2700000 } +# timeout is in milliseconds: 15 minutes, the Launch plan's maximum. +params = { useProxy = true, solveCaptcha = true, timeout = 900000 } diff --git a/benchmarks/retail-cart-providers.toml b/benchmarks/retail-cart-providers.toml index 61f9841..b6825a9 100644 --- a/benchmarks/retail-cart-providers.toml +++ b/benchmarks/retail-cart-providers.toml @@ -7,4 +7,11 @@ arms = [ "codex-pw-steel", "codex-pw-hyperbrowser", "codex-pw-anchor", + "codex-pw-browser-use", + "claude-pw-kernel", + "claude-pw-kernel-macos", + "claude-pw-browserbase", + "claude-pw-browser-use", + "claude-pw-hyperbrowser", + "claude-pw-steel", ] diff --git a/benchmarks/retail-cart.toml b/benchmarks/retail-cart.toml index c980f3d..d6bfc24 100644 --- a/benchmarks/retail-cart.toml +++ b/benchmarks/retail-cart.toml @@ -1,5 +1,6 @@ # Guest add-to-cart flows on bot-protected retail and industrial-supply sites, # driven through the browser REPL, with a stealth browser and with the config -# registry's per-site settings, and with a fixed per-site table. +# registry's per-site settings, and with a fixed per-site table, plus the +# stealth arm with Claude Code in place of Codex. suite = "retail-cart" -arms = ["codex-repl", "codex-repl-site-config", "codex-repl-pinned"] +arms = ["codex-repl", "codex-repl-site-config", "codex-repl-pinned", "claude-repl"] diff --git a/configurators/pinned/configure b/configurators/pinned/configure index b5ec6f8..6762441 100755 --- a/configurators/pinned/configure +++ b/configurators/pinned/configure @@ -25,7 +25,12 @@ def main() -> int: site = tomllib.loads(task["config"]).get("metadata", {}).get("site") entry = TABLE["sites"].get(site) if entry is None: - raise SystemExit(f"No pinned configuration for site {site!r}") + # Sites added after the table was measured run on the arm's static config. + json.dump( + {"browser": {}, "steering": None, "provenance": {"site": site, "pinned": False}}, + sys.stdout, + ) + return 0 browser = dict(entry["browser"]) browser["proxy_create"] = {"type": entry["proxy"], "config": {"country": "US"}} persona = entry.get("persona") diff --git a/harness/definitions.py b/harness/definitions.py index 00da835..f8880e5 100644 --- a/harness/definitions.py +++ b/harness/definitions.py @@ -15,6 +15,7 @@ ENVIRONMENTS = { "kernel": "harness.environment:KernelTaskEnvironment", "remote-browser": "harness.environment:RemoteBrowserEnvironment", + "kernel-vm-remote-browser": "harness.environment:KernelVmRemoteBrowserEnvironment", } @@ -32,7 +33,8 @@ class Arm: model: str toolset: str # "kernel" runs the task container in the Kernel browser's VM; - # "remote-browser" runs it on local Docker and drives the browser over CDP. + # "remote-browser" runs it on local Docker and drives the browser over CDP; + # "kernel-vm-remote-browser" does the same on a Kernel VM used as compute. environment: str = "kernel" agent_version: str | None = None agent_kwargs: dict[str, Any] = field(default_factory=dict) diff --git a/harness/environment.py b/harness/environment.py index df773f3..3da4011 100644 --- a/harness/environment.py +++ b/harness/environment.py @@ -25,6 +25,8 @@ and drives a browser from any provider in `harness.providers` over CDP, so providers are compared with the same agent, tools and container. Its config file names the `provider` and that provider's session create `params`. +`KernelVmRemoteBrowserEnvironment` does the same with the task container on a +Kernel VM used only as compute, for machines without Docker. """ from __future__ import annotations @@ -227,17 +229,18 @@ async def stop(self, delete: bool) -> None: self.logger.error(f"Failed to delete Kernel proxy {self._proxy_id}: {error}") -class RemoteBrowserEnvironment(DockerEnvironment): - def __init__(self, environment_dir: Path, *args: Any, **kwargs: Any) -> None: +class _RemoteBrowser: + """Create the trial's browser from the config file's provider, write its CDP + URL into the task container, and release it when the trial ends.""" + + def _read_remote_config(self, environment_dir: Path) -> None: config = json.loads((Path(environment_dir) / CONFIG_FILE).read_text()) self._provider_name: str = config["provider"] self._provider = providers.get(self._provider_name) self._create_params: dict[str, Any] = config.get("params") or {} self._browser_session_id: str | None = None - super().__init__(environment_dir, *args, **kwargs) - async def start(self, force_build: bool) -> None: - await super().start(force_build) + async def _attach_remote_browser(self) -> None: try: session = await self._provider.create(self._create_params) except Exception as error: @@ -258,16 +261,53 @@ async def start(self, force_build: bool) -> None: if result.return_code != 0: raise BrowserCreationError(f"Could not write {CDP_URL_FILE}: {result.stderr}") + async def _release_remote_browser(self) -> None: + if self._browser_session_id is None: + return + try: + await self._provider.release(self._browser_session_id) + self._browser_session_id = None + except Exception as error: + self.logger.error( + f"Failed to release {self._provider_name} browser " + f"{self._browser_session_id}: {error}" + ) + + +class RemoteBrowserEnvironment(_RemoteBrowser, DockerEnvironment): + """The task container on local Docker, driving the provider's browser over CDP.""" + + def __init__(self, environment_dir: Path, *args: Any, **kwargs: Any) -> None: + self._read_remote_config(environment_dir) + super().__init__(environment_dir, *args, **kwargs) + + async def start(self, force_build: bool) -> None: + await super().start(force_build) + await self._attach_remote_browser() + async def stop(self, delete: bool) -> None: try: await super().stop(delete) finally: - if self._browser_session_id is not None: - try: - await self._provider.release(self._browser_session_id) - self._browser_session_id = None - except Exception as error: - self.logger.error( - f"Failed to release {self._provider_name} browser " - f"{self._browser_session_id}: {error}" - ) + await self._release_remote_browser() + + +class KernelVmRemoteBrowserEnvironment(_RemoteBrowser, KernelEnvironment): + """The task container on a Kernel VM used only as compute (no local Docker + needed), driving the provider's browser over CDP like `RemoteBrowserEnvironment`. + The VM's own browser is left unused, so a Kernel arm is reached over the + network the same way as every other provider.""" + + def __init__(self, environment_dir: Path, *args: Any, **kwargs: Any) -> None: + self._read_remote_config(environment_dir) + super().__init__(environment_dir, *args, **kwargs) + + async def start(self, force_build: bool) -> None: + await super().start(force_build) + await self._attach_remote_browser() + + async def stop(self, delete: bool) -> None: + try: + await super().stop(delete) + finally: + await self._release_remote_browser() diff --git a/harness/providers.py b/harness/providers.py index 9326bfa..ab66573 100644 --- a/harness/providers.py +++ b/harness/providers.py @@ -9,7 +9,9 @@ from __future__ import annotations +import inspect import os +import urllib.parse from dataclasses import dataclass from typing import Any @@ -38,7 +40,12 @@ class KernelProvider(Provider): async def create(self, params: dict[str, Any]) -> Session: async with AsyncKernel() as client: - browser = await client.browsers.create(**params) + # Params the SDK does not know yet (such as the closed-beta + # `fingerprint`) are sent in the request body as they are. + known = inspect.signature(client.browsers.create).parameters + kwargs = {key: value for key, value in params.items() if key in known} + extra = {key: value for key, value in params.items() if key not in known} + browser = await client.browsers.create(**kwargs, extra_body=extra or None) return Session(browser.session_id, browser.cdp_ws_url) async def release(self, session_id: str) -> None: @@ -82,9 +89,14 @@ class SteelProvider(HttpProvider): base_url = "https://api.steel.dev/v1" key_header = "steel-api-key" + def _project(self) -> dict[str, str]: + # An organization-scoped key must name the project in each request body. + project = os.environ.get("STEEL_PROJECT_ID") + return {"projectId": project} if project else {} + async def create(self, params: dict[str, Any]) -> Session: async with self._client() as client: - response = await client.post("/sessions", json=params) + response = await client.post("/sessions", json={**self._project(), **params}) response.raise_for_status() session = response.json() url = f"wss://connect.steel.dev?apiKey={os.environ['STEEL_API_KEY']}&sessionId={session['id']}" @@ -92,7 +104,8 @@ async def create(self, params: dict[str, Any]) -> Session: async def release(self, session_id: str) -> None: async with self._client() as client: - (await client.post(f"/sessions/{session_id}/release")).raise_for_status() + response = await client.post(f"/sessions/{session_id}/release", json=self._project()) + response.raise_for_status() class HyperbrowserProvider(HttpProvider): @@ -105,13 +118,37 @@ async def create(self, params: dict[str, Any]) -> Session: response = await client.post("/session", json=params) response.raise_for_status() session = response.json() - return Session(session["id"], session["wsEndpoint"]) + # Hyperbrowser stops a session when its CDP client disconnects unless the + # URL asks to keep it alive, and each trial connects three times. + url = session["wsEndpoint"] + url += ("&" if urllib.parse.urlsplit(url).query else "?") + "keepAlive=true" + return Session(session["id"], url) async def release(self, session_id: str) -> None: async with self._client() as client: (await client.put(f"/session/{session_id}/stop")).raise_for_status() +class BrowserUseProvider(HttpProvider): + env = ("BROWSER_USE_API_KEY",) + base_url = "https://api.browser-use.com/api/v4" + key_header = "X-Browser-Use-API-Key" + + async def create(self, params: dict[str, Any]) -> Session: + async with self._client() as client: + response = await client.post("/browsers", json=params) + response.raise_for_status() + session = response.json() + if not session.get("cdpUrl"): + raise RuntimeError(f"Browser Use returned no CDP URL for browser {session.get('id')}") + return Session(session["id"], session["cdpUrl"]) + + async def release(self, session_id: str) -> None: + async with self._client() as client: + response = await client.patch(f"/browsers/{session_id}", json={"action": "stop"}) + response.raise_for_status() + + class AnchorProvider(HttpProvider): env = ("ANCHOR_API_KEY",) base_url = "https://api.anchorbrowser.io/v1" @@ -134,6 +171,7 @@ async def release(self, session_id: str) -> None: "browserbase": BrowserbaseProvider(), "steel": SteelProvider(), "hyperbrowser": HyperbrowserProvider(), + "browser-use": BrowserUseProvider(), "anchor": AnchorProvider(), } diff --git a/harness/run.py b/harness/run.py index a1aa941..b046007 100644 --- a/harness/run.py +++ b/harness/run.py @@ -112,7 +112,7 @@ def run( benchmark_name, arm_names, task_ids, work_dir, toolset_ref, suite_args, toolset_args ) for item in prepared: - if item.arm.environment == "remote-browser": + if item.arm.environment in ("remote-browser", "kernel-vm-remote-browser"): for configuration in item.configurations.values(): preflight.require_env(*providers.get(configuration["browser"]["provider"]).env) diff --git a/suites/retail-cart/image/bench/score b/suites/retail-cart/image/bench/score index 4bcff6e..894f8ad 100644 --- a/suites/retail-cart/image/bench/score +++ b/suites/retail-cart/image/bench/score @@ -39,6 +39,12 @@ FAILURES = ("failed", "blocked", "website_error", "kernel_error") FINAL_PAGE = "return { url: page.url(), title: await page.title() };" # A bot check or access-denied page in place of the cart. +# An empty cart page can still list the item in recommendations ("You may also like"). +EMPTY_CART = re.compile( + r"your (shopping )?(bag|cart|basket) is (currently )?empty|(bag|cart|basket) is empty", + re.IGNORECASE, +) + CHALLENGE = re.compile( r"captcha|/checkpoint/|validateCaptcha|verify (yourself|access|you are)|access denied" r"|access is temporarily restricted|press (and|&) hold|are you a robot", @@ -130,7 +136,10 @@ def check_cart(task: dict, final_page: dict) -> dict: return {"verified": False, "unverifiable": True, "error": "task has no cart_url or cart_api"} text = seen.pop("text", None) or "" match = re.search(task["cart_match"], text, re.IGNORECASE) if task.get("cart_match") else None - seen["verified"] = match is not None + empty = EMPTY_CART.search(text[:5000]) + seen["verified"] = match is not None and empty is None + if empty: + seen["empty_cart"] = empty.group(0) challenge = CHALLENGE.search(f"{seen.get('url', '')} {seen.get('title', '')} {text[:3000]}") seen["unverifiable"] = match is None and ("error" in seen or challenge is not None) if challenge and match is None: diff --git a/suites/retail-cart/suite.py b/suites/retail-cart/suite.py index 879c005..7d4c602 100644 --- a/suites/retail-cart/suite.py +++ b/suites/retail-cart/suite.py @@ -53,6 +53,9 @@ class Task: cart_url: str | None = None cart_api: str | None = None timeout_sec: int = 900 + # easy: little or no bot protection, a control. standard: protected sites + # most browsers get through. hard: sites that block most automated browsers. + tier: str = "standard" def preflight() -> None: @@ -63,8 +66,24 @@ def load_tasks() -> list[Task]: return [Task(**entry) for entry in tomllib.loads((HERE / "tasks.toml").read_text())["tasks"]] -def generate(task_ids: list[str] | None, out_dir: Path) -> list[Path]: +TIERS = ("easy", "standard", "hard") + + +def generate( + task_ids: list[str] | None, out_dir: Path, tiers: list[str] | str | None = None +) -> list[Path]: + """Write the tasks with these ids, or every task in `tiers` (default: all tiers). + + `tiers` is a list or a comma-separated string, so `--suite-arg tiers=hard` works. + """ tasks = load_tasks() + if isinstance(tiers, str): + tiers = [tier.strip() for tier in tiers.split(",") if tier.strip()] + if tiers: + unknown = sorted(set(tiers) - set(TIERS)) + if unknown: + raise SystemExit(f"Unknown tiers: {', '.join(unknown)}; known: {', '.join(TIERS)}") + tasks = [task for task in tasks if task.tier in tiers] if task_ids: known = {task.id: task for task in tasks} missing = [task_id for task_id in task_ids if task_id not in known] @@ -106,8 +125,8 @@ def _write(task: Task, task_dir: Path) -> Path: config = tomlkit.document() config["schema_version"] = "1.3" config["source"] = "retail-cart" - config["task"] = {"name": f"retail-cart/{task.id}", "keywords": [task.site]} - config["metadata"] = {"site": task.site, "start_url": task.start_url} + config["task"] = {"name": f"retail-cart/{task.id}", "keywords": [task.site, task.tier]} + config["metadata"] = {"site": task.site, "start_url": task.start_url, "tier": task.tier} config["environment"] = { "build_timeout_sec": 1200.0, "network_mode": "public", diff --git a/suites/retail-cart/tasks.toml b/suites/retail-cart/tasks.toml index 233d945..3c09d7e 100644 --- a/suites/retail-cart/tasks.toml +++ b/suites/retail-cart/tasks.toml @@ -2,6 +2,10 @@ # where most bot protection first sees the browser. The verifier opens # `cart_url` (or fetches `cart_api` from the page) and checks it lists the # item with `cart_match`, a case-insensitive regular expression. +# +# `tier` groups the sites (default `standard`): `easy` sites have little or no +# bot protection and act as controls; `hard` sites block most automated +# browsers. Run one tier with `--suite-arg tiers=hard`. [[tasks]] id = "amazon-aa-batteries" @@ -91,6 +95,56 @@ item = "a HydroJug 32 oz sport bottle that can be shipped" cart_url = "https://www.dickssportinggoods.com/OrderItemDisplay" cart_match = 'HydroJug' +[[tasks]] +id = "zoro-nitrile-gloves" +site = "zoro.com" +start_url = "https://www.zoro.com/" +item = "a box of disposable nitrile gloves, size large" +cart_url = "https://www.zoro.com/cart" +cart_match = 'nitrile' + +# Standard: protected sites, added for vendor coverage. Wayfair is hard: its +# press-and-hold check stopped Kernel with the macOS fingerprint on first try. + +[[tasks]] +id = "wayfair-hangers" +site = "wayfair.com" +start_url = "https://www.wayfair.com/" +item = "a pack of clothes hangers that can be shipped" +cart_url = "https://www.wayfair.com/v/checkout/basket/show" +cart_match = 'hanger' +tier = "hard" + +[[tasks]] +id = "zara-socks" +site = "zara.com" +start_url = "https://www.zara.com/us/" +item = "one pack of socks" +cart_url = "https://www.zara.com/us/en/shop/cart" +cart_match = 'sock' + +# Easy: controls with little bot protection. + +[[tasks]] +id = "ikea-side-table" +site = "ikea.com" +start_url = "https://www.ikea.com/us/en/" +item = "one LACK side table that can be delivered" +cart_url = "https://www.ikea.com/us/en/shoppingcart/" +cart_match = 'LACK\b.{0,80}side table|side table.{0,80}LACK\b' +tier = "easy" + +[[tasks]] +id = "gymshark-socks" +site = "gymshark.com" +start_url = "https://www.gymshark.com/" +item = "one pack of Gymshark crew socks" +cart_url = "https://www.gymshark.com/cart" +cart_match = 'sock' +tier = "easy" + +# Hard: sites that block most automated browsers. + [[tasks]] id = "mouser-usb-c-receptacle" site = "mouser.com" @@ -98,6 +152,7 @@ start_url = "https://www.mouser.com/" item = "one GCT USB4105-GF-A USB-C receptacle" cart_url = "https://www.mouser.com/en/Cart/" cart_match = 'USB4105' +tier = "hard" [[tasks]] id = "newark-aa-batteries" @@ -106,14 +161,7 @@ start_url = "https://www.newark.com/" item = "a pack of AA alkaline batteries" cart_url = "https://www.newark.com/webapp/wcs/stores/servlet/OrderItemDisplay?storeId=10194&catalogId=15003&langId=-1" cart_match = 'alkaline' - -[[tasks]] -id = "zoro-nitrile-gloves" -site = "zoro.com" -start_url = "https://www.zoro.com/" -item = "a box of disposable nitrile gloves, size large" -cart_url = "https://www.zoro.com/cart" -cart_match = 'nitrile' +tier = "hard" [[tasks]] id = "guitarcenter-picks" @@ -122,6 +170,7 @@ start_url = "https://www.guitarcenter.com/" item = "a 12-pack of Fender 351 celluloid medium guitar picks" cart_api = "https://www.guitarcenter.com/rest/model/atg/commerce/ShoppingCartActor/cartRedesignNew?saveForLaterPage=1&SPA=true" cart_match = '\b351\b' +tier = "hard" [[tasks]] id = "wbmason-sticky-notes" @@ -130,3 +179,77 @@ start_url = "https://www.wbmason.com/" item = "a pack of Post-it 3 x 3 in canary yellow sticky notes" cart_url = "https://www.wbmason.com/shopping-cart" cart_match = 'Post-it|sticky note' +tier = "hard" + +[[tasks]] +id = "adidas-socks" +site = "adidas.com" +start_url = "https://www.adidas.com/us" +item = "one pack of adidas crew socks" +cart_url = "https://www.adidas.com/us/cart" +cart_match = 'sock' +tier = "hard" + +[[tasks]] +id = "bestbuy-hdmi-cable" +site = "bestbuy.com" +start_url = "https://www.bestbuy.com/" +item = "an Insignia HDMI cable that can be shipped" +cart_url = "https://www.bestbuy.com/cart" +cart_match = 'Insignia.{0,120}HDMI|HDMI.{0,120}Insignia' +tier = "hard" + +[[tasks]] +id = "sephora-niacinamide-serum" +site = "sephora.com" +start_url = "https://www.sephora.com/" +item = "one The Ordinary Niacinamide 10% + Zinc 1% serum that can be shipped" +cart_url = "https://www.sephora.com/basket" +# Specific enough not to match the basket page's promotions and recommendations. +cart_match = 'Niacinamide 10%\s*\+\s*Zinc' +tier = "hard" + +[[tasks]] +id = "nike-socks" +site = "nike.com" +start_url = "https://www.nike.com/" +item = "one pack of Nike Everyday crew socks" +cart_url = "https://www.nike.com/cart" +cart_match = 'Everyday|sock' +tier = "hard" + +[[tasks]] +id = "footlocker-socks" +site = "footlocker.com" +start_url = "https://www.footlocker.com/" +item = "one pack of crew socks" +cart_url = "https://www.footlocker.com/cart" +cart_match = 'sock' +tier = "hard" + +[[tasks]] +id = "chewy-dental-treats" +site = "chewy.com" +start_url = "https://www.chewy.com/" +item = "a bag or box of Greenies dental dog treats" +cart_url = "https://www.chewy.com/app/cart" +cart_match = 'Greenies' +tier = "hard" + +[[tasks]] +id = "bhphoto-microsd" +site = "bhphotovideo.com" +start_url = "https://www.bhphotovideo.com/" +item = "a SanDisk microSD memory card" +cart_url = "https://www.bhphotovideo.com/find/cart.jsp" +cart_match = 'SanDisk' +tier = "hard" + +[[tasks]] +id = "kohls-bath-towel" +site = "kohls.com" +start_url = "https://www.kohls.com/" +item = "one bath towel that can be shipped" +cart_url = "https://www.kohls.com/checkout/shopping_cart.jsp" +cart_match = 'towel' +tier = "hard" diff --git a/tests/test_definitions.py b/tests/test_definitions.py index 30fba7e..2b672c6 100644 --- a/tests/test_definitions.py +++ b/tests/test_definitions.py @@ -19,7 +19,7 @@ def test_provider_arms_run_on_remote_browsers(): benchmark = load_benchmark("retail-cart-providers") for name in benchmark.arms: arm = load_arm(name) - assert arm.environment == "remote-browser" + assert arm.environment in ("remote-browser", "kernel-vm-remote-browser") assert arm.toolset == "playwright-mcp" assert arm.browser.config["provider"] in providers.PROVIDERS diff --git a/tests/test_environment.py b/tests/test_environment.py index 0f0b523..4bbf392 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -215,9 +215,12 @@ async def release(self, session_id): self.calls.append(("release", session_id)) -def _remote_environment(tmp_path, monkeypatch, provider): +def _remote_environment(tmp_path, monkeypatch, provider, cls=None): from harbor.environments.base import ExecResult - from harbor.environments.docker.docker import DockerEnvironment + + cls = cls or environment.RemoteBrowserEnvironment + # The Harbor environment class the remote browser is layered on. + base = next(base for base in cls.__mro__[1:] if base.__module__.startswith("harbor.")) calls = [] @@ -235,15 +238,15 @@ def init(self, environment_dir, **kwargs): self._persistent_env = {} self.logger = logging.getLogger("test") - monkeypatch.setattr(DockerEnvironment, "__init__", init) - monkeypatch.setattr(DockerEnvironment, "start", start) - monkeypatch.setattr(DockerEnvironment, "stop", stop) - monkeypatch.setattr(DockerEnvironment, "exec", exec) + monkeypatch.setattr(base, "__init__", init) + monkeypatch.setattr(base, "start", start) + monkeypatch.setattr(base, "stop", stop) + monkeypatch.setattr(base, "exec", exec) monkeypatch.setitem(environment.providers.PROVIDERS, "fake", provider) (tmp_path / "kernel-config.json").write_text( json.dumps({"provider": "fake", "params": {"stealth": True}}) ) - return environment.RemoteBrowserEnvironment(environment_dir=tmp_path), calls + return cls(environment_dir=tmp_path), calls def test_remote_browser_is_created_after_the_container_and_released_after(tmp_path, monkeypatch): @@ -267,3 +270,18 @@ def test_remote_browser_creation_failure_is_identifiable(tmp_path, monkeypatch): env, _ = _remote_environment(tmp_path, monkeypatch, FakeProvider(fail=True)) with pytest.raises(environment.BrowserCreationError, match="no capacity"): asyncio.run(env.start(force_build=False)) + + +def test_kernel_vm_remote_browser_uses_the_vm_only_as_compute(tmp_path, monkeypatch): + provider = FakeProvider() + env, calls = _remote_environment( + tmp_path, monkeypatch, provider, environment.KernelVmRemoteBrowserEnvironment + ) + + asyncio.run(env.start(force_build=False)) + asyncio.run(env.stop(delete=True)) + + assert calls[0] == "container started" + assert environment.CDP_URL_FILE in calls[1][0] and calls[1][1] == {"CDP_URL": "wss://remote/1"} + assert calls[2] == "container stopped" + assert provider.calls == [("create", {"stealth": True}), ("release", "remote-1")] diff --git a/tests/test_pinned.py b/tests/test_pinned.py index 157ee6f..9288471 100644 --- a/tests/test_pinned.py +++ b/tests/test_pinned.py @@ -14,12 +14,21 @@ def _configure(site): return json.loads(result.stdout) -def test_every_retail_cart_site_has_a_pinned_configuration(): +def test_measured_sites_are_pinned_and_new_sites_fall_back(): + import tomllib + from harness.definitions import load_suite + measured = tomllib.loads((CONFIGURE.parent / "sites.toml").read_text())["sites"] for task in load_suite("retail-cart").load_tasks(): output = _configure(task.site) - assert output["browser"]["proxy_create"]["config"] == {"country": "US"} + if task.site in measured: + assert output["browser"]["proxy_create"]["config"] == {"country": "US"} + else: + # Not yet measured by hand: the arm's static config, recorded as unpinned. + assert output["browser"] == {} and output["provenance"]["pinned"] is False + # The original 16 sites were all measured. + assert len(measured) >= 16 def test_personas_become_chromium_env(): diff --git a/tests/test_providers.py b/tests/test_providers.py index 9a9cbdf..ce41973 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -56,6 +56,62 @@ def test_steel_connects_with_the_session_and_key(monkeypatch): assert [request.url.path for request in requests] == ["/v1/sessions", "/v1/sessions/s1/release"] +def test_steel_names_the_project_for_an_organization_key(monkeypatch): + monkeypatch.setenv("STEEL_API_KEY", "sk") + monkeypatch.setenv("STEEL_PROJECT_ID", "sp") + provider = providers.get("steel") + requests = _serve(monkeypatch, provider, lambda request: httpx.Response(200, json={"id": "s1"})) + + asyncio.run(provider.create({"useProxy": True})) + asyncio.run(provider.release("s1")) + + assert json.loads(requests[0].content) == {"projectId": "sp", "useProxy": True} + assert json.loads(requests[1].content) == {"projectId": "sp"} + + +def test_browser_use_creates_and_stops_a_browser(monkeypatch): + provider = providers.get("browser-use") + requests = _serve( + monkeypatch, + provider, + lambda request: httpx.Response(200, json={"id": "bu1", "cdpUrl": "wss://bu/1"}), + ) + + session = asyncio.run(provider.create({"proxyCountryCode": "us"})) + asyncio.run(provider.release(session.id)) + + assert session == providers.Session("bu1", "wss://bu/1") + assert json.loads(requests[0].content) == {"proxyCountryCode": "us"} + assert requests[1].method == "PATCH" and requests[1].url.path == "/api/v4/browsers/bu1" + assert json.loads(requests[1].content) == {"action": "stop"} + + +def test_kernel_sends_params_the_sdk_does_not_know_in_the_body(monkeypatch): + calls = [] + + class Browsers: + async def create(self, *, stealth=None, timeout_seconds=None, extra_body=None): + calls.append({"stealth": stealth, "extra_body": extra_body}) + return type("Browser", (), {"session_id": "k1", "cdp_ws_url": "wss://k/1"}) + + class Client: + browsers = Browsers() + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return None + + monkeypatch.setattr(providers, "AsyncKernel", Client) + session = asyncio.run( + providers.get("kernel").create({"stealth": True, "fingerprint": {"os": "macos"}}) + ) + + assert session == providers.Session("k1", "wss://k/1") + assert calls == [{"stealth": True, "extra_body": {"fingerprint": {"os": "macos"}}}] + + def test_hyperbrowser_and_anchor_read_their_cdp_urls(monkeypatch): hyperbrowser = providers.get("hyperbrowser") _serve( @@ -70,7 +126,10 @@ def test_hyperbrowser_and_anchor_read_their_cdp_urls(monkeypatch): lambda request: httpx.Response(200, json={"data": {"id": "a1", "cdp_url": "wss://a/1"}}), ) - assert asyncio.run(hyperbrowser.create({})) == providers.Session("h1", "wss://hb/1") + # Hyperbrowser stops a session on CDP disconnect unless the URL keeps it alive. + assert asyncio.run(hyperbrowser.create({})) == providers.Session( + "h1", "wss://hb/1?keepAlive=true" + ) assert asyncio.run(anchor.create({})) == providers.Session("a1", "wss://a/1") diff --git a/tests/test_retail_cart.py b/tests/test_retail_cart.py index 962c9a5..37544dc 100644 --- a/tests/test_retail_cart.py +++ b/tests/test_retail_cart.py @@ -32,6 +32,19 @@ def test_generate_writes_a_harbor_task(tmp_path): assert (task_dir / "environment/bench/finalize-mcp").is_file() +def test_tiers_filter_tasks_and_are_recorded(tmp_path): + tasks = suite.load_tasks() + assert {task.tier for task in tasks} == set(suite.TIERS) + hard = [task.id for task in tasks if task.tier == "hard"] + task_dirs = suite.generate(None, tmp_path, tiers="hard") + assert sorted(path.name for path in task_dirs) == sorted(hard) + config = tomllib.loads((task_dirs[0] / "task.toml").read_text()) + assert config["metadata"]["tier"] == "hard" + assert len(suite.generate(None, tmp_path / "all")) == len(tasks) + with pytest.raises(SystemExit, match="Unknown tiers: extreme"): + suite.generate(None, tmp_path, tiers=["extreme"]) + + def test_unknown_task_is_refused(tmp_path): with pytest.raises(SystemExit, match="nope"): suite.generate(["nope"], tmp_path)