diff --git a/.github/workflows/_run.yml b/.github/workflows/_run.yml index e0ee4b5..701c8be 100644 --- a/.github/workflows/_run.yml +++ b/.github/workflows/_run.yml @@ -65,6 +65,14 @@ jobs: CLAWBENCH_JUDGE_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} CLAWBENCH_JUDGE_MODEL: deepseek/deepseek-v4-pro CLAWBENCH_JUDGE_API_TYPE: openai-completions + # Other browser providers, for remote-browser arms. + BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} + BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} + STEEL_API_KEY: ${{ secrets.STEEL_API_KEY }} + STEEL_PROJECT_ID: ${{ secrets.STEEL_PROJECT_ID }} + BROWSER_USE_API_KEY: ${{ secrets.BROWSER_USE_API_KEY }} + HYPERBROWSER_API_KEY: ${{ secrets.HYPERBROWSER_API_KEY }} + ANCHOR_API_KEY: ${{ secrets.ANCHOR_API_KEY }} BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} BRAINTRUST_PROJECT: ${{ vars.BRAINTRUST_PROJECT }} steps: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72ff36b..828bfaa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,8 @@ jobs: run: uv run bench prepare kernel-mcp-clawbench --arms codex-kernel-mcp,claude-kernel-mcp --tasks v2-1134-chapter-finder-redcross --out "$RUNNER_TEMP/prepared" - name: Prepare the static retail-cart arm run: uv run bench prepare retail-cart --arms codex-repl --out "$RUNNER_TEMP/prepared-retail-cart" + - name: Prepare the retail-cart provider arms + run: uv run bench prepare retail-cart-providers --out "$RUNNER_TEMP/prepared-retail-cart-providers" report: runs-on: ubuntu-latest diff --git a/.github/workflows/retail-cart-providers.yml b/.github/workflows/retail-cart-providers.yml new file mode 100644 index 0000000..affec33 --- /dev/null +++ b/.github/workflows/retail-cart-providers.yml @@ -0,0 +1,45 @@ +name: Retail cart flows on each provider's browser + +on: + workflow_dispatch: + inputs: + arms: + description: Comma-separated arm names (empty runs every arm in benchmarks/retail-cart-providers.toml) + required: false + default: "" + type: string + tasks: + description: Comma-separated task IDs, or all + required: false + default: all + type: string + attempts: + description: Trials per task and arm + required: false + default: "10" + type: string + concurrency: + description: Concurrent trials per arm. Every arm's task containers run on the one runner. + required: false + default: "4" + type: string + playwright_mcp_version: + description: "@playwright/mcp version to install (empty installs the toolset's pinned version)" + required: false + default: "" + type: string + +permissions: + contents: read + +jobs: + benchmark: + uses: ./.github/workflows/_run.yml + secrets: inherit + with: + benchmark: retail-cart-providers + arms: ${{ inputs.arms }} + tasks: ${{ inputs.tasks }} + attempts: ${{ inputs.attempts }} + concurrency: ${{ inputs.concurrency }} + toolset_ref: ${{ inputs.playwright_mcp_version }} diff --git a/README.md b/README.md index 17942f6..efefce0 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ A run of two arms on two tasks (Red Cross and Indeed) produced this summary: | `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` | Codex and Claude Code arms; the `site-config` configurator's model | | `PURELY_MAIL_API_KEY`, `PURELY_MAIL_DOMAIN` | ClawBench creates a disposable [PurelyMail](https://purelymail.com) mailbox on your domain for every task, so agents can sign up and receive confirmation emails. | | `CLAWBENCH_JUDGE_BASE_URL`, `CLAWBENCH_JUDGE_API_KEY`, `CLAWBENCH_JUDGE_MODEL`, `CLAWBENCH_JUDGE_API_TYPE` | ClawBench's LLM judge, which checks the intercepted request against the task. Any OpenAI-compatible endpoint works with `CLAWBENCH_JUDGE_API_TYPE=openai-completions`; we use `deepseek/deepseek-v4-pro` through OpenRouter. See [ClawBench](https://github.com/TIGER-AI-Lab/ClawBench) for details. | +| `BROWSERBASE_API_KEY`, `BROWSERBASE_PROJECT_ID`, `BROWSER_USE_API_KEY`, `STEEL_API_KEY` (plus `STEEL_PROJECT_ID` for an organization-scoped key), `HYPERBROWSER_API_KEY`, `ANCHOR_API_KEY` | Arms on those providers' browsers (`environment = "remote-browser"` or `"kernel-vm-remote-browser"`); checked before a run that includes them | | `BRAINTRUST_API_KEY`, `BRAINTRUST_PROJECT` | Optional: publishing results | ## Running in GitHub Actions @@ -121,7 +122,7 @@ ClawBench's primary score is the lenient reward. `kernel_mcp_valid` records whet ### `retail-cart` -Guest add-to-cart tasks on 16 retail and industrial-supply sites with heavy bot protection. Each task starts on the site's home page and asks the agent to put one item in the site's cart as a guest, by whatever route works, and show the cart. +Guest add-to-cart tasks on 28 retail and industrial-supply sites, in three tiers: 2 `easy` controls with little bot protection, 13 `standard` protected sites, and 13 `hard` sites that block most automated browsers. Run one tier with `--suite-arg tiers=hard`. Each task starts on the site's home page and asks the agent to put one item in the site's cart as a guest, by whatever route works, and show the cart. | Arm | Agent | Browser | |---|---|---| @@ -139,6 +140,28 @@ Run several attempts per task to get a pass rate per site: uv run bench run retail-cart --attempts 10 ``` +### `retail-cart-providers` + +The `retail-cart` tasks on each browser provider's browser. Within an agent, every arm uses the same tools ([Playwright MCP](https://github.com/microsoft/playwright-mcp) attached over CDP) and the same task container, so arms differ only in the browser. The `codex-pw-*` arms (Codex, `gpt-6-luna`) run the container on the runner's Docker (`environment = "remote-browser"`). The `claude-pw-*` arms (Claude Code, `claude-sonnet-5-5`) run it on a Kernel VM used only as compute (`environment = "kernel-vm-remote-browser"`), which needs no local Docker; the Kernel arms still get a separate browser session, reached over the network like every other provider's. Each provider gets its own anti-bot settings: + +| Provider | Arms | Browser | +|---|---|---| +| Kernel | `codex-pw-kernel`, `claude-pw-kernel` | `stealth: true` (managed ISP proxy and CAPTCHA solver) | +| Kernel, macOS | `claude-pw-kernel-macos` | as above, plus the macOS fingerprint (`fingerprint: {os: macos}`, closed beta) | +| Browserbase | `codex-pw-browserbase`, `claude-pw-browserbase` | CAPTCHA solving and its residential proxies. Advanced Stealth is left out: Browserbase rejects it outside the Enterprise plan | +| Browser Use | `codex-pw-browser-use`, `claude-pw-browser-use` | its US proxy | +| Steel | `codex-pw-steel`, `claude-pw-steel` | residential proxy and CAPTCHA solving; 15-minute sessions, the Launch plan's maximum | +| Hyperbrowser | `codex-pw-hyperbrowser`, `claude-pw-hyperbrowser` | stealth, proxy and CAPTCHA solving | +| Anchor | `codex-pw-anchor` | its US proxy, extra stealth and CAPTCHA solving | + +The start page, the agent and the verifier each connect to the browser separately, so sessions are created to outlive a CDP disconnect (Browserbase's `keepAlive`, Hyperbrowser's `keepAlive=true` on the CDP URL, Anchor's idle timeout). A Steel key scoped to the whole organization also needs `STEEL_PROJECT_ID`. Scoring is the same as `retail-cart`. + +```bash +uv run bench run retail-cart-providers --attempts 10 --concurrency 4 +``` + +All arms' task containers run on one machine, so keep `--concurrency` low on a GitHub runner. + ## Extending ### Add an arm @@ -162,6 +185,8 @@ steering = true `toolset_args` are passed to the toolset; `kernel-mcp-local` takes `toolsets`, the Kernel MCP toolsets to enable: `playwright` (the default), `repl`, `computer`, or several. `agent_kwargs` are passed to the Harbor agent. +An arm with `environment = "remote-browser"` runs the task container on local Docker instead of in the Kernel browser's VM, and its `config` names a `provider` from `harness/providers.py` and that provider's session create `params`. The environment writes the browser's CDP URL to `/run/browser/cdp-url` in the container; the `playwright-mcp` toolset connects to it. + `config` takes any [`POST /browsers`](https://kernel.sh/docs/api-reference/browsers/create-a-browser-session) parameter, such as `stealth`, `proxy`, `region`, `extensions`, or `viewport`. Browsers are always headful. ### Add a configurator diff --git a/arms/claude-pw-browser-use.toml b/arms/claude-pw-browser-use.toml new file mode 100644 index 0000000..47b4b36 --- /dev/null +++ b/arms/claude-pw-browser-use.toml @@ -0,0 +1,16 @@ +# Claude Code driving a browser-use browser through Playwright MCP over CDP, the +# same agent, tools and task container as every claude-pw-* arm. The container +# runs on a Kernel VM used only as compute; the browser is a separate session. +# Browser: Browser Use's cloud browser with its managed US proxy. +agent = "claude-code" +model = "claude-sonnet-5-5" +agent_version = "2.1.238" +toolset = "playwright-mcp" +environment = "kernel-vm-remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "browser-use" +params = { proxyCountryCode = "us", timeout = 30 } diff --git a/arms/claude-pw-browserbase.toml b/arms/claude-pw-browserbase.toml new file mode 100644 index 0000000..c5e79e4 --- /dev/null +++ b/arms/claude-pw-browserbase.toml @@ -0,0 +1,17 @@ +# Claude Code driving a browserbase browser through Playwright MCP over CDP, the +# same agent, tools and task container as every claude-pw-* arm. The container +# runs on a Kernel VM used only as compute; the browser is a separate session. +# Browser: Browserbase's managed proxies and CAPTCHA solving. Advanced stealth is left +# out: Browserbase rejects it outside the Enterprise plan ("Verified mode"). +agent = "claude-code" +model = "claude-sonnet-5-5" +agent_version = "2.1.238" +toolset = "playwright-mcp" +environment = "kernel-vm-remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "browserbase" +params = { keepAlive = true, timeout = 1800, region = "us-east-1", proxies = true, browserSettings = { solveCaptchas = true } } diff --git a/arms/claude-pw-hyperbrowser.toml b/arms/claude-pw-hyperbrowser.toml new file mode 100644 index 0000000..c059310 --- /dev/null +++ b/arms/claude-pw-hyperbrowser.toml @@ -0,0 +1,16 @@ +# Claude Code driving a hyperbrowser browser through Playwright MCP over CDP, the +# same agent, tools and task container as every claude-pw-* arm. The container +# runs on a Kernel VM used only as compute; the browser is a separate session. +# Browser: Hyperbrowser's stealth, managed US proxy and CAPTCHA solving. +agent = "claude-code" +model = "claude-sonnet-5-5" +agent_version = "2.1.238" +toolset = "playwright-mcp" +environment = "kernel-vm-remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "hyperbrowser" +params = { useStealth = true, useProxy = true, proxyCountry = "US", solveCaptchas = true, timeoutMinutes = 30 } diff --git a/arms/claude-pw-kernel-macos.toml b/arms/claude-pw-kernel-macos.toml new file mode 100644 index 0000000..67860bc --- /dev/null +++ b/arms/claude-pw-kernel-macos.toml @@ -0,0 +1,17 @@ +# Claude Code driving a kernel browser through Playwright MCP over CDP, the +# same agent, tools and task container as every claude-pw-* arm. The container +# runs on a Kernel VM used only as compute; the browser is a separate session. +# Browser: Kernel's anti-detection mode plus its macOS fingerprint (closed beta): macOS +# user agent and platform, Apple GPU renderer. +agent = "claude-code" +model = "claude-sonnet-5-5" +agent_version = "2.1.238" +toolset = "playwright-mcp" +environment = "kernel-vm-remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "kernel" +params = { stealth = true, timeout_seconds = 1800, fingerprint = { os = "macos" } } diff --git a/arms/claude-pw-kernel.toml b/arms/claude-pw-kernel.toml new file mode 100644 index 0000000..8ebff5f --- /dev/null +++ b/arms/claude-pw-kernel.toml @@ -0,0 +1,16 @@ +# Claude Code driving a kernel browser through Playwright MCP over CDP, the +# same agent, tools and task container as every claude-pw-* arm. The container +# runs on a Kernel VM used only as compute; the browser is a separate session. +# Browser: Kernel's anti-detection mode (`stealth`): managed ISP proxy and CAPTCHA solver. +agent = "claude-code" +model = "claude-sonnet-5-5" +agent_version = "2.1.238" +toolset = "playwright-mcp" +environment = "kernel-vm-remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "kernel" +params = { stealth = true, timeout_seconds = 1800 } diff --git a/arms/claude-pw-steel.toml b/arms/claude-pw-steel.toml new file mode 100644 index 0000000..f07016a --- /dev/null +++ b/arms/claude-pw-steel.toml @@ -0,0 +1,17 @@ +# Claude Code driving a steel browser through Playwright MCP over CDP, the +# same agent, tools and task container as every claude-pw-* arm. The container +# runs on a Kernel VM used only as compute; the browser is a separate session. +# Browser: Steel's managed residential proxy and CAPTCHA solving. 15-minute sessions: the +# Launch plan's maximum (other arms get 30), so a long trial can lose its browser. +agent = "claude-code" +model = "claude-sonnet-5-5" +agent_version = "2.1.238" +toolset = "playwright-mcp" +environment = "kernel-vm-remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "steel" +params = { useProxy = true, solveCaptcha = true, timeout = 900000 } diff --git a/arms/claude-repl.toml b/arms/claude-repl.toml new file mode 100644 index 0000000..e07e233 --- /dev/null +++ b/arms/claude-repl.toml @@ -0,0 +1,16 @@ +# Same as codex-repl, but with Claude Code driving the browser only through +# Kernel MCP's browser REPL, with a fixed stealth browser. +agent = "claude-code" +model = "claude-sonnet-5-5" +agent_version = "2.1.238" +toolset = "kernel-mcp-local" + +# Claude Code's own web tools would let the agent answer without the browser. +[agent_kwargs] +disable_web_search = true + +[toolset_args] +toolsets = ["repl"] + +[browser] +config = { stealth = true } diff --git a/arms/codex-pw-anchor.toml b/arms/codex-pw-anchor.toml new file mode 100644 index 0000000..3ac96c3 --- /dev/null +++ b/arms/codex-pw-anchor.toml @@ -0,0 +1,24 @@ +# Codex driving an Anchor browser through Playwright MCP over CDP, with +# Anchor's anti-bot settings: its US proxy, extra stealth and CAPTCHA solving +# (both need the proxy). +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "anchor" + +[browser.config.params.session] +proxy = { active = true, type = "anchor_proxy", country_code = "us" } +# Minutes. The idle timeout covers the gaps between the start page, agent and +# verifier connections. +timeout = { max_duration = 45, idle_timeout = 30 } + +[browser.config.params.browser] +extra_stealth = { active = true } +captcha_solver = { active = true } diff --git a/arms/codex-pw-browser-use.toml b/arms/codex-pw-browser-use.toml new file mode 100644 index 0000000..659dd1c --- /dev/null +++ b/arms/codex-pw-browser-use.toml @@ -0,0 +1,14 @@ +# Codex driving a Browser Use cloud browser through Playwright MCP over CDP, with +# Browser Use's managed US proxy. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "browser-use" +params = { proxyCountryCode = "us", timeout = 45 } diff --git a/arms/codex-pw-browserbase.toml b/arms/codex-pw-browserbase.toml new file mode 100644 index 0000000..ef045a0 --- /dev/null +++ b/arms/codex-pw-browserbase.toml @@ -0,0 +1,22 @@ +# Codex driving a Browserbase browser through Playwright MCP over CDP, with +# Browserbase's anti-bot settings: CAPTCHA solving and its residential proxies. +# Advanced Stealth is left out: Browserbase rejects it outside the Enterprise +# plan ("Verified mode is only available on the Enterprise plan"). keepAlive keeps the session across the +# start page, agent and verifier connections. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "browserbase" + +[browser.config.params] +keepAlive = true +timeout = 2700 +proxies = true +browserSettings = { solveCaptchas = true } diff --git a/arms/codex-pw-hyperbrowser.toml b/arms/codex-pw-hyperbrowser.toml new file mode 100644 index 0000000..3a6879c --- /dev/null +++ b/arms/codex-pw-hyperbrowser.toml @@ -0,0 +1,14 @@ +# Codex driving a Hyperbrowser browser through Playwright MCP over CDP, with +# Hyperbrowser's anti-bot settings: stealth, proxy and CAPTCHA solving. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "hyperbrowser" +params = { useStealth = true, useProxy = true, solveCaptchas = true, timeoutMinutes = 45 } diff --git a/arms/codex-pw-kernel.toml b/arms/codex-pw-kernel.toml new file mode 100644 index 0000000..bc9b4c4 --- /dev/null +++ b/arms/codex-pw-kernel.toml @@ -0,0 +1,14 @@ +# Codex driving a Kernel stealth browser through Playwright MCP over CDP, the +# same agent, tools and container as the other providers' codex-pw-* arms. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "kernel" +params = { stealth = true, timeout_seconds = 1800 } diff --git a/arms/codex-pw-steel.toml b/arms/codex-pw-steel.toml new file mode 100644 index 0000000..0550218 --- /dev/null +++ b/arms/codex-pw-steel.toml @@ -0,0 +1,15 @@ +# Codex driving a Steel browser through Playwright MCP over CDP, with Steel's +# anti-bot settings: residential proxy and CAPTCHA solving. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "steel" +# timeout is in milliseconds: 15 minutes, the Launch plan's maximum. +params = { useProxy = true, solveCaptcha = true, timeout = 900000 } diff --git a/benchmarks/retail-cart-providers.toml b/benchmarks/retail-cart-providers.toml new file mode 100644 index 0000000..b6825a9 --- /dev/null +++ b/benchmarks/retail-cart-providers.toml @@ -0,0 +1,17 @@ +# The retail-cart tasks on each provider's browser, with the same agent, +# Playwright MCP tools and task container, so arms differ only in the browser. +suite = "retail-cart" +arms = [ + "codex-pw-kernel", + "codex-pw-browserbase", + "codex-pw-steel", + "codex-pw-hyperbrowser", + "codex-pw-anchor", + "codex-pw-browser-use", + "claude-pw-kernel", + "claude-pw-kernel-macos", + "claude-pw-browserbase", + "claude-pw-browser-use", + "claude-pw-hyperbrowser", + "claude-pw-steel", +] diff --git a/benchmarks/retail-cart.toml b/benchmarks/retail-cart.toml index c980f3d..d6bfc24 100644 --- a/benchmarks/retail-cart.toml +++ b/benchmarks/retail-cart.toml @@ -1,5 +1,6 @@ # Guest add-to-cart flows on bot-protected retail and industrial-supply sites, # driven through the browser REPL, with a stealth browser and with the config -# registry's per-site settings, and with a fixed per-site table. +# registry's per-site settings, and with a fixed per-site table, plus the +# stealth arm with Claude Code in place of Codex. suite = "retail-cart" -arms = ["codex-repl", "codex-repl-site-config", "codex-repl-pinned"] +arms = ["codex-repl", "codex-repl-site-config", "codex-repl-pinned", "claude-repl"] diff --git a/configurators/pinned/configure b/configurators/pinned/configure index b5ec6f8..6762441 100755 --- a/configurators/pinned/configure +++ b/configurators/pinned/configure @@ -25,7 +25,12 @@ def main() -> int: site = tomllib.loads(task["config"]).get("metadata", {}).get("site") entry = TABLE["sites"].get(site) if entry is None: - raise SystemExit(f"No pinned configuration for site {site!r}") + # Sites added after the table was measured run on the arm's static config. + json.dump( + {"browser": {}, "steering": None, "provenance": {"site": site, "pinned": False}}, + sys.stdout, + ) + return 0 browser = dict(entry["browser"]) browser["proxy_create"] = {"type": entry["proxy"], "config": {"country": "US"}} persona = entry.get("persona") diff --git a/harness/definitions.py b/harness/definitions.py index 57e12d4..f8880e5 100644 --- a/harness/definitions.py +++ b/harness/definitions.py @@ -11,6 +11,12 @@ from typing import Any ROOT = Path(__file__).resolve().parents[1] +# Harbor environment each arm `environment` runs its trials in. +ENVIRONMENTS = { + "kernel": "harness.environment:KernelTaskEnvironment", + "remote-browser": "harness.environment:RemoteBrowserEnvironment", + "kernel-vm-remote-browser": "harness.environment:KernelVmRemoteBrowserEnvironment", +} @dataclass(frozen=True) @@ -26,6 +32,10 @@ class Arm: agent: str model: str toolset: str + # "kernel" runs the task container in the Kernel browser's VM; + # "remote-browser" runs it on local Docker and drives the browser over CDP; + # "kernel-vm-remote-browser" does the same on a Kernel VM used as compute. + environment: str = "kernel" agent_version: str | None = None agent_kwargs: dict[str, Any] = field(default_factory=dict) browser: Browser = field(default_factory=Browser) @@ -59,7 +69,10 @@ def load_benchmark(name: str) -> Benchmark: def load_arm(name: str) -> Arm: data = _read(ROOT / "arms" / f"{name}.toml") browser = data.pop("browser", {}) - return Arm(name=name, browser=Browser(**browser), **data) + arm = Arm(name=name, browser=Browser(**browser), **data) + if arm.environment not in ENVIRONMENTS: + raise SystemExit(f"Arm {name} has unknown environment {arm.environment!r}") + return arm def load_suite(name: str) -> ModuleType: diff --git a/harness/environment.py b/harness/environment.py index bf47afc..3da4011 100644 --- a/harness/environment.py +++ b/harness/environment.py @@ -20,6 +20,13 @@ the persona or WebGL renderer they ask for. `vm_setup` is a list of shell commands then run as root. A trial where either fails fails with `VmSetupError`. + +`RemoteBrowserEnvironment` instead runs the task container on local Docker +and drives a browser from any provider in `harness.providers` over CDP, so +providers are compared with the same agent, tools and container. Its config +file names the `provider` and that provider's session create `params`. +`KernelVmRemoteBrowserEnvironment` does the same with the task container on a +Kernel VM used only as compute, for machines without Docker. """ from __future__ import annotations @@ -33,14 +40,20 @@ from pathlib import Path from typing import Any +from harbor.environments.docker.docker import DockerEnvironment from harbor.environments.kernel import KernelEnvironment from kernel import AsyncKernel, Kernel +from harness import providers + CONFIG_FILE = "kernel-config.json" PROXY_CREATE = "proxy_create" CHROMIUM_ENV = "chromium_env" VM_SETUP = "vm_setup" RUN_ID_ENV = "BENCH_RUN_ID" +# Where the task container finds the remote browser's CDP URL. Agents may not +# pass their environment on to the MCP servers they launch, so it is a file. +CDP_URL_FILE = "/run/browser/cdp-url" class ProxyCreationError(RuntimeError): @@ -51,6 +64,10 @@ class VmSetupError(RuntimeError): """The browser's VM could not be set up for the trial before the first page load.""" +class BrowserCreationError(RuntimeError): + """The provider could not create the trial's remote browser.""" + + SUPERVISOR_PROGRAM = """[program:chromium] command=/usr/bin/env {env} /usr/local/bin/chromium-launcher autostart=false @@ -210,3 +227,87 @@ async def stop(self, delete: bool) -> None: self._proxy_id = None except Exception as error: self.logger.error(f"Failed to delete Kernel proxy {self._proxy_id}: {error}") + + +class _RemoteBrowser: + """Create the trial's browser from the config file's provider, write its CDP + URL into the task container, and release it when the trial ends.""" + + def _read_remote_config(self, environment_dir: Path) -> None: + config = json.loads((Path(environment_dir) / CONFIG_FILE).read_text()) + self._provider_name: str = config["provider"] + self._provider = providers.get(self._provider_name) + self._create_params: dict[str, Any] = config.get("params") or {} + self._browser_session_id: str | None = None + + async def _attach_remote_browser(self) -> None: + try: + session = await self._provider.create(self._create_params) + except Exception as error: + raise BrowserCreationError( + f"Could not create a {self._provider_name} browser: {error}" + ) from error + self._browser_session_id = session.id + self.logger.info(f"Created {self._provider_name} browser {session.id}") + self._persistent_env.update( + {"BROWSER_PROVIDER": self._provider_name, "BROWSER_SESSION_ID": session.id} + ) + directory = os.path.dirname(CDP_URL_FILE) + result = await self.exec( + f'mkdir -p {directory} && printf %s "$CDP_URL" > {CDP_URL_FILE} && chmod 0644 {CDP_URL_FILE}', + env={"CDP_URL": session.cdp_url}, + user="root", + ) + if result.return_code != 0: + raise BrowserCreationError(f"Could not write {CDP_URL_FILE}: {result.stderr}") + + async def _release_remote_browser(self) -> None: + if self._browser_session_id is None: + return + try: + await self._provider.release(self._browser_session_id) + self._browser_session_id = None + except Exception as error: + self.logger.error( + f"Failed to release {self._provider_name} browser " + f"{self._browser_session_id}: {error}" + ) + + +class RemoteBrowserEnvironment(_RemoteBrowser, DockerEnvironment): + """The task container on local Docker, driving the provider's browser over CDP.""" + + def __init__(self, environment_dir: Path, *args: Any, **kwargs: Any) -> None: + self._read_remote_config(environment_dir) + super().__init__(environment_dir, *args, **kwargs) + + async def start(self, force_build: bool) -> None: + await super().start(force_build) + await self._attach_remote_browser() + + async def stop(self, delete: bool) -> None: + try: + await super().stop(delete) + finally: + await self._release_remote_browser() + + +class KernelVmRemoteBrowserEnvironment(_RemoteBrowser, KernelEnvironment): + """The task container on a Kernel VM used only as compute (no local Docker + needed), driving the provider's browser over CDP like `RemoteBrowserEnvironment`. + The VM's own browser is left unused, so a Kernel arm is reached over the + network the same way as every other provider.""" + + def __init__(self, environment_dir: Path, *args: Any, **kwargs: Any) -> None: + self._read_remote_config(environment_dir) + super().__init__(environment_dir, *args, **kwargs) + + async def start(self, force_build: bool) -> None: + await super().start(force_build) + await self._attach_remote_browser() + + async def stop(self, delete: bool) -> None: + try: + await super().stop(delete) + finally: + await self._release_remote_browser() diff --git a/harness/providers.py b/harness/providers.py new file mode 100644 index 0000000..ab66573 --- /dev/null +++ b/harness/providers.py @@ -0,0 +1,182 @@ +"""Cloud browser providers that a trial can drive over CDP. + +Each provider creates one browser session from an arm's create params and +releases it when the trial ends. The params are the provider's own session +create body, so an arm states exactly which stealth, proxy and CAPTCHA +settings it asked for. Sessions must outlive a CDP disconnect: the start page +is opened, the agent connects, and the verifier connects, each separately. +""" + +from __future__ import annotations + +import inspect +import os +import urllib.parse +from dataclasses import dataclass +from typing import Any + +import httpx +from kernel import AsyncKernel + + +@dataclass(frozen=True) +class Session: + id: str + cdp_url: str + + +class Provider: + env: tuple[str, ...] = () + + async def create(self, params: dict[str, Any]) -> Session: + raise NotImplementedError + + async def release(self, session_id: str) -> None: + raise NotImplementedError + + +class KernelProvider(Provider): + env = ("KERNEL_API_KEY",) + + async def create(self, params: dict[str, Any]) -> Session: + async with AsyncKernel() as client: + # Params the SDK does not know yet (such as the closed-beta + # `fingerprint`) are sent in the request body as they are. + known = inspect.signature(client.browsers.create).parameters + kwargs = {key: value for key, value in params.items() if key in known} + extra = {key: value for key, value in params.items() if key not in known} + browser = await client.browsers.create(**kwargs, extra_body=extra or None) + return Session(browser.session_id, browser.cdp_ws_url) + + async def release(self, session_id: str) -> None: + async with AsyncKernel() as client: + await client.browsers.delete_by_id(session_id) + + +class HttpProvider(Provider): + base_url: str + key_header: str + + def _client(self) -> httpx.AsyncClient: + return httpx.AsyncClient( + base_url=self.base_url, + headers={self.key_header: os.environ[self.env[0]]}, + timeout=120, + ) + + +class BrowserbaseProvider(HttpProvider): + env = ("BROWSERBASE_API_KEY", "BROWSERBASE_PROJECT_ID") + base_url = "https://api.browserbase.com/v1" + key_header = "X-BB-API-Key" + + async def create(self, params: dict[str, Any]) -> Session: + body = {"projectId": os.environ["BROWSERBASE_PROJECT_ID"], **params} + async with self._client() as client: + response = await client.post("/sessions", json=body) + response.raise_for_status() + session = response.json() + return Session(session["id"], session["connectUrl"]) + + async def release(self, session_id: str) -> None: + body = {"projectId": os.environ["BROWSERBASE_PROJECT_ID"], "status": "REQUEST_RELEASE"} + async with self._client() as client: + (await client.post(f"/sessions/{session_id}", json=body)).raise_for_status() + + +class SteelProvider(HttpProvider): + env = ("STEEL_API_KEY",) + base_url = "https://api.steel.dev/v1" + key_header = "steel-api-key" + + def _project(self) -> dict[str, str]: + # An organization-scoped key must name the project in each request body. + project = os.environ.get("STEEL_PROJECT_ID") + return {"projectId": project} if project else {} + + async def create(self, params: dict[str, Any]) -> Session: + async with self._client() as client: + response = await client.post("/sessions", json={**self._project(), **params}) + response.raise_for_status() + session = response.json() + url = f"wss://connect.steel.dev?apiKey={os.environ['STEEL_API_KEY']}&sessionId={session['id']}" + return Session(session["id"], url) + + async def release(self, session_id: str) -> None: + async with self._client() as client: + response = await client.post(f"/sessions/{session_id}/release", json=self._project()) + response.raise_for_status() + + +class HyperbrowserProvider(HttpProvider): + env = ("HYPERBROWSER_API_KEY",) + base_url = "https://api.hyperbrowser.ai/api" + key_header = "x-api-key" + + async def create(self, params: dict[str, Any]) -> Session: + async with self._client() as client: + response = await client.post("/session", json=params) + response.raise_for_status() + session = response.json() + # Hyperbrowser stops a session when its CDP client disconnects unless the + # URL asks to keep it alive, and each trial connects three times. + url = session["wsEndpoint"] + url += ("&" if urllib.parse.urlsplit(url).query else "?") + "keepAlive=true" + return Session(session["id"], url) + + async def release(self, session_id: str) -> None: + async with self._client() as client: + (await client.put(f"/session/{session_id}/stop")).raise_for_status() + + +class BrowserUseProvider(HttpProvider): + env = ("BROWSER_USE_API_KEY",) + base_url = "https://api.browser-use.com/api/v4" + key_header = "X-Browser-Use-API-Key" + + async def create(self, params: dict[str, Any]) -> Session: + async with self._client() as client: + response = await client.post("/browsers", json=params) + response.raise_for_status() + session = response.json() + if not session.get("cdpUrl"): + raise RuntimeError(f"Browser Use returned no CDP URL for browser {session.get('id')}") + return Session(session["id"], session["cdpUrl"]) + + async def release(self, session_id: str) -> None: + async with self._client() as client: + response = await client.patch(f"/browsers/{session_id}", json={"action": "stop"}) + response.raise_for_status() + + +class AnchorProvider(HttpProvider): + env = ("ANCHOR_API_KEY",) + base_url = "https://api.anchorbrowser.io/v1" + key_header = "anchor-api-key" + + async def create(self, params: dict[str, Any]) -> Session: + async with self._client() as client: + response = await client.post("/sessions", json=params) + response.raise_for_status() + session = response.json()["data"] + return Session(session["id"], session["cdp_url"]) + + async def release(self, session_id: str) -> None: + async with self._client() as client: + (await client.delete(f"/sessions/{session_id}")).raise_for_status() + + +PROVIDERS: dict[str, Provider] = { + "kernel": KernelProvider(), + "browserbase": BrowserbaseProvider(), + "steel": SteelProvider(), + "hyperbrowser": HyperbrowserProvider(), + "browser-use": BrowserUseProvider(), + "anchor": AnchorProvider(), +} + + +def get(name: str) -> Provider: + if name not in PROVIDERS: + raise SystemExit(f"Unknown browser provider {name!r}; known: {', '.join(PROVIDERS)}") + return PROVIDERS[name] diff --git a/harness/run.py b/harness/run.py index f18b53f..b046007 100644 --- a/harness/run.py +++ b/harness/run.py @@ -11,9 +11,16 @@ from pathlib import Path from typing import Any -from harness import preflight +from harness import preflight, providers from harness.configure import configure_tasks -from harness.definitions import Arm, load_arm, load_benchmark, load_suite, load_toolset +from harness.definitions import ( + ENVIRONMENTS, + Arm, + load_arm, + load_benchmark, + load_suite, + load_toolset, +) from harness.environment import RUN_ID_ENV, delete_run_proxies # Transient failures worth retrying as a whole trial: Kernel API hiccups and @@ -104,6 +111,10 @@ def run( prepared = prepare( benchmark_name, arm_names, task_ids, work_dir, toolset_ref, suite_args, toolset_args ) + for item in prepared: + if item.arm.environment in ("remote-browser", "kernel-vm-remote-browser"): + for configuration in item.configurations.values(): + preflight.require_env(*providers.get(configuration["browser"]["provider"]).env) jobs_dir.mkdir(parents=True, exist_ok=True) manifest_path = jobs_dir / f"run-{run_id}.json" @@ -146,7 +157,7 @@ def harbor_command( "--path", str(item.dataset), "--agent", arm.agent, "--model", arm.model, - "--env", "harness.environment:KernelTaskEnvironment", + "--env", ENVIRONMENTS[arm.environment], "--job-name", job_name, "--jobs-dir", str(jobs_dir), "--n-concurrent", str(concurrency), diff --git a/suites/retail-cart/image/Dockerfile b/suites/retail-cart/image/Dockerfile index 4362d77..78620f0 100644 --- a/suites/retail-cart/image/Dockerfile +++ b/suites/retail-cart/image/Dockerfile @@ -1,13 +1,18 @@ # Task image for retail-cart. The browser is the Kernel VM this container runs -# in, so the image only holds the finalize tool, the start-URL setup and the -# scorer. Toolsets append their own layers. +# in, or a remote browser reached over CDP, so the image only holds the +# finalize tool, the start-URL setup and the scorer. Toolsets append their own +# layers. FROM node:22-bookworm-slim RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates curl git python3 \ && rm -rf /var/lib/apt/lists/* +# cdp-execute drives a remote browser; it needs no browser of its own. +RUN npm install --global playwright-core@1.64.1 +ENV NODE_PATH=/usr/local/lib/node_modules + COPY bench/ /opt/bench/ -RUN chmod +x /opt/bench/finalize-mcp /opt/bench/open-start-url /opt/bench/score +RUN chmod +x /opt/bench/cdp-execute /opt/bench/finalize-mcp /opt/bench/open-start-url /opt/bench/score WORKDIR /app diff --git a/suites/retail-cart/image/bench/cdp-execute b/suites/retail-cart/image/bench/cdp-execute new file mode 100755 index 0000000..17010ba --- /dev/null +++ b/suites/retail-cart/image/bench/cdp-execute @@ -0,0 +1,34 @@ +#!/usr/bin/env node +// Run Playwright code against the trial's remote browser over CDP, the way +// Kernel's playwright/execute API runs it: the code on stdin is the body of an +// async function given `page`, `context` and `browser`. Prints +// {success, result} or {success: false, error} as JSON. +const fs = require("fs"); +const { chromium } = require("playwright-core"); + +const CDP_URL_FILE = "/run/browser/cdp-url"; + +async function main() { + const code = fs.readFileSync(0, "utf8"); + const timeoutMs = Number(process.argv[2] || 60) * 1000; + const timer = setTimeout(() => { + console.log(JSON.stringify({ success: false, error: `timed out after ${timeoutMs} ms` })); + process.exit(0); + }, timeoutMs); + const browser = await chromium.connectOverCDP(fs.readFileSync(CDP_URL_FILE, "utf8").trim()); + const context = browser.contexts()[0] ?? (await browser.newContext()); + // The page the agent was last working in: the newest one that is not blank. + const pages = context.pages().filter((p) => p.url() !== "about:blank"); + const page = pages.at(-1) ?? context.pages()[0] ?? (await context.newPage()); + const run = new (Object.getPrototypeOf(async () => {}).constructor)("page", "context", "browser", code); + const result = await run(page, context, browser); + clearTimeout(timer); + console.log(JSON.stringify({ success: true, result: result ?? null })); + // Exit without closing the browser: the session belongs to the trial. + process.exit(0); +} + +main().catch((error) => { + console.log(JSON.stringify({ success: false, error: String(error && error.message ? error.message : error) })); + process.exit(0); +}); diff --git a/suites/retail-cart/image/bench/finalize-mcp b/suites/retail-cart/image/bench/finalize-mcp index 9339d7a..3d07f23 100755 --- a/suites/retail-cart/image/bench/finalize-mcp +++ b/suites/retail-cart/image/bench/finalize-mcp @@ -35,8 +35,8 @@ OUTCOMES = { "to load." ), "kernel_error": ( - "the Kernel browser or its tools kept you from finishing: the browser REPL or the " - "browser itself errored, hung, or crashed." + "the browser or its tools kept you from finishing: the browser tools or the browser " + "itself errored, hung, or crashed." ), } diff --git a/suites/retail-cart/image/bench/open-start-url b/suites/retail-cart/image/bench/open-start-url index d6dd7f3..3ee95f5 100755 --- a/suites/retail-cart/image/bench/open-start-url +++ b/suites/retail-cart/image/bench/open-start-url @@ -9,10 +9,15 @@ from __future__ import annotations import json import os +import subprocess import sys import time import urllib.error import urllib.request +from pathlib import Path + +# Written by the remote-browser environment; absent in a Kernel browser's VM. +CDP_URL_FILE = Path("/run/browser/cdp-url") CODE = """ await page.goto(%s, { waitUntil: "domcontentloaded", timeout: 45000 }); @@ -22,10 +27,29 @@ return page.url(); def main(url: str) -> int: + if CDP_URL_FILE.exists(): + output = subprocess.run( + ["/opt/bench/cdp-execute", "90"], + input=CODE % json.dumps(url), + capture_output=True, + text=True, + timeout=120, + ) + result = json.loads(output.stdout) + else: + result = execute_with_kernel(CODE % json.dumps(url)) + if not result.get("success"): + print(f"Could not open {url}: {result.get('error')}", file=sys.stderr) + return 1 + print(f"Opened {result.get('result')}") + return 0 + + +def execute_with_kernel(code: str) -> dict: base = (os.environ.get("KERNEL_BASE_URL") or "https://api.onkernel.com").rstrip("/") request = urllib.request.Request( f"{base}/browsers/{os.environ['KERNEL_SESSION_ID']}/playwright/execute", - data=json.dumps({"code": CODE % json.dumps(url), "timeout_sec": 90}).encode(), + data=json.dumps({"code": code, "timeout_sec": 90}).encode(), headers={ "Authorization": f"Bearer {os.environ['KERNEL_API_KEY']}", "Content-Type": "application/json", @@ -44,11 +68,7 @@ def main(url: str) -> int: if attempt == 4: raise time.sleep(2 * (attempt + 1)) - if not result.get("success"): - print(f"Could not open {url}: {result.get('error')}", file=sys.stderr) - return 1 - print(f"Opened {result.get('result')}") - return 0 + return result if __name__ == "__main__": diff --git a/suites/retail-cart/image/bench/score b/suites/retail-cart/image/bench/score index 175d4a8..894f8ad 100644 --- a/suites/retail-cart/image/bench/score +++ b/suites/retail-cart/image/bench/score @@ -7,7 +7,9 @@ finalize, and `outcome_` which failure it reported, so bot blocks can be told apart from other website errors, Kernel errors and the agent's own failures. -The browser's final URL and title are saved, then the verifier checks the +The browser is the Kernel VM's, reached through Kernel's API, or a remote +browser reached over CDP when the environment wrote its URL. The browser's +final URL and title are saved, then the verifier checks the cart itself: it reads the page the agent ended on when that is already the cart, otherwise opens the task's `cart_url`, or fetches its `cart_api` from the page, and sets `cart_verified` when the text matches `cart_match`. When @@ -22,18 +24,27 @@ import json import os import re import shutil +import subprocess import urllib.parse import urllib.request from pathlib import Path REPORT = Path("/logs/agent/finalize.json") TASK = Path("/opt/bench/task.json") +# Written by the remote-browser environment; absent in a Kernel browser's VM. +CDP_URL_FILE = Path("/run/browser/cdp-url") VERIFIER = Path("/logs/verifier") FAILURES = ("failed", "blocked", "website_error", "kernel_error") FINAL_PAGE = "return { url: page.url(), title: await page.title() };" # A bot check or access-denied page in place of the cart. +# An empty cart page can still list the item in recommendations ("You may also like"). +EMPTY_CART = re.compile( + r"your (shopping )?(bag|cart|basket) is (currently )?empty|(bag|cart|basket) is empty", + re.IGNORECASE, +) + CHALLENGE = re.compile( r"captcha|/checkpoint/|validateCaptcha|verify (yourself|access|you are)|access denied" r"|access is temporarily restricted|press (and|&) hold|are you a robot", @@ -69,6 +80,8 @@ return await page.evaluate(async (url) => { def execute(code: str, timeout_sec: int) -> dict: + if CDP_URL_FILE.exists(): + return execute_over_cdp(code, timeout_sec) base = (os.environ.get("KERNEL_BASE_URL") or "https://api.onkernel.com").rstrip("/") request = urllib.request.Request( f"{base}/browsers/{os.environ['KERNEL_SESSION_ID']}/playwright/execute", @@ -89,6 +102,23 @@ def execute(code: str, timeout_sec: int) -> dict: return result.get("result") or {} +def execute_over_cdp(code: str, timeout_sec: int) -> dict: + try: + output = subprocess.run( + ["/opt/bench/cdp-execute", str(timeout_sec)], + input=code, + capture_output=True, + text=True, + timeout=timeout_sec + 30, + ) + result = json.loads(output.stdout) + except (OSError, ValueError, subprocess.SubprocessError) as error: + return {"error": str(error)} + if not result.get("success"): + return {"error": result.get("error")} + return result.get("result") or {} + + def same_page(url: str, other: str) -> bool: a, b = urllib.parse.urlsplit(url), urllib.parse.urlsplit(other) return (a.netloc, a.path.rstrip("/").lower()) == (b.netloc, b.path.rstrip("/").lower()) @@ -106,7 +136,10 @@ def check_cart(task: dict, final_page: dict) -> dict: return {"verified": False, "unverifiable": True, "error": "task has no cart_url or cart_api"} text = seen.pop("text", None) or "" match = re.search(task["cart_match"], text, re.IGNORECASE) if task.get("cart_match") else None - seen["verified"] = match is not None + empty = EMPTY_CART.search(text[:5000]) + seen["verified"] = match is not None and empty is None + if empty: + seen["empty_cart"] = empty.group(0) challenge = CHALLENGE.search(f"{seen.get('url', '')} {seen.get('title', '')} {text[:3000]}") seen["unverifiable"] = match is None and ("error" in seen or challenge is not None) if challenge and match is None: diff --git a/suites/retail-cart/suite.py b/suites/retail-cart/suite.py index 879c005..7d4c602 100644 --- a/suites/retail-cart/suite.py +++ b/suites/retail-cart/suite.py @@ -53,6 +53,9 @@ class Task: cart_url: str | None = None cart_api: str | None = None timeout_sec: int = 900 + # easy: little or no bot protection, a control. standard: protected sites + # most browsers get through. hard: sites that block most automated browsers. + tier: str = "standard" def preflight() -> None: @@ -63,8 +66,24 @@ def load_tasks() -> list[Task]: return [Task(**entry) for entry in tomllib.loads((HERE / "tasks.toml").read_text())["tasks"]] -def generate(task_ids: list[str] | None, out_dir: Path) -> list[Path]: +TIERS = ("easy", "standard", "hard") + + +def generate( + task_ids: list[str] | None, out_dir: Path, tiers: list[str] | str | None = None +) -> list[Path]: + """Write the tasks with these ids, or every task in `tiers` (default: all tiers). + + `tiers` is a list or a comma-separated string, so `--suite-arg tiers=hard` works. + """ tasks = load_tasks() + if isinstance(tiers, str): + tiers = [tier.strip() for tier in tiers.split(",") if tier.strip()] + if tiers: + unknown = sorted(set(tiers) - set(TIERS)) + if unknown: + raise SystemExit(f"Unknown tiers: {', '.join(unknown)}; known: {', '.join(TIERS)}") + tasks = [task for task in tasks if task.tier in tiers] if task_ids: known = {task.id: task for task in tasks} missing = [task_id for task_id in task_ids if task_id not in known] @@ -106,8 +125,8 @@ def _write(task: Task, task_dir: Path) -> Path: config = tomlkit.document() config["schema_version"] = "1.3" config["source"] = "retail-cart" - config["task"] = {"name": f"retail-cart/{task.id}", "keywords": [task.site]} - config["metadata"] = {"site": task.site, "start_url": task.start_url} + config["task"] = {"name": f"retail-cart/{task.id}", "keywords": [task.site, task.tier]} + config["metadata"] = {"site": task.site, "start_url": task.start_url, "tier": task.tier} config["environment"] = { "build_timeout_sec": 1200.0, "network_mode": "public", diff --git a/suites/retail-cart/tasks.toml b/suites/retail-cart/tasks.toml index 233d945..3c09d7e 100644 --- a/suites/retail-cart/tasks.toml +++ b/suites/retail-cart/tasks.toml @@ -2,6 +2,10 @@ # where most bot protection first sees the browser. The verifier opens # `cart_url` (or fetches `cart_api` from the page) and checks it lists the # item with `cart_match`, a case-insensitive regular expression. +# +# `tier` groups the sites (default `standard`): `easy` sites have little or no +# bot protection and act as controls; `hard` sites block most automated +# browsers. Run one tier with `--suite-arg tiers=hard`. [[tasks]] id = "amazon-aa-batteries" @@ -91,6 +95,56 @@ item = "a HydroJug 32 oz sport bottle that can be shipped" cart_url = "https://www.dickssportinggoods.com/OrderItemDisplay" cart_match = 'HydroJug' +[[tasks]] +id = "zoro-nitrile-gloves" +site = "zoro.com" +start_url = "https://www.zoro.com/" +item = "a box of disposable nitrile gloves, size large" +cart_url = "https://www.zoro.com/cart" +cart_match = 'nitrile' + +# Standard: protected sites, added for vendor coverage. Wayfair is hard: its +# press-and-hold check stopped Kernel with the macOS fingerprint on first try. + +[[tasks]] +id = "wayfair-hangers" +site = "wayfair.com" +start_url = "https://www.wayfair.com/" +item = "a pack of clothes hangers that can be shipped" +cart_url = "https://www.wayfair.com/v/checkout/basket/show" +cart_match = 'hanger' +tier = "hard" + +[[tasks]] +id = "zara-socks" +site = "zara.com" +start_url = "https://www.zara.com/us/" +item = "one pack of socks" +cart_url = "https://www.zara.com/us/en/shop/cart" +cart_match = 'sock' + +# Easy: controls with little bot protection. + +[[tasks]] +id = "ikea-side-table" +site = "ikea.com" +start_url = "https://www.ikea.com/us/en/" +item = "one LACK side table that can be delivered" +cart_url = "https://www.ikea.com/us/en/shoppingcart/" +cart_match = 'LACK\b.{0,80}side table|side table.{0,80}LACK\b' +tier = "easy" + +[[tasks]] +id = "gymshark-socks" +site = "gymshark.com" +start_url = "https://www.gymshark.com/" +item = "one pack of Gymshark crew socks" +cart_url = "https://www.gymshark.com/cart" +cart_match = 'sock' +tier = "easy" + +# Hard: sites that block most automated browsers. + [[tasks]] id = "mouser-usb-c-receptacle" site = "mouser.com" @@ -98,6 +152,7 @@ start_url = "https://www.mouser.com/" item = "one GCT USB4105-GF-A USB-C receptacle" cart_url = "https://www.mouser.com/en/Cart/" cart_match = 'USB4105' +tier = "hard" [[tasks]] id = "newark-aa-batteries" @@ -106,14 +161,7 @@ start_url = "https://www.newark.com/" item = "a pack of AA alkaline batteries" cart_url = "https://www.newark.com/webapp/wcs/stores/servlet/OrderItemDisplay?storeId=10194&catalogId=15003&langId=-1" cart_match = 'alkaline' - -[[tasks]] -id = "zoro-nitrile-gloves" -site = "zoro.com" -start_url = "https://www.zoro.com/" -item = "a box of disposable nitrile gloves, size large" -cart_url = "https://www.zoro.com/cart" -cart_match = 'nitrile' +tier = "hard" [[tasks]] id = "guitarcenter-picks" @@ -122,6 +170,7 @@ start_url = "https://www.guitarcenter.com/" item = "a 12-pack of Fender 351 celluloid medium guitar picks" cart_api = "https://www.guitarcenter.com/rest/model/atg/commerce/ShoppingCartActor/cartRedesignNew?saveForLaterPage=1&SPA=true" cart_match = '\b351\b' +tier = "hard" [[tasks]] id = "wbmason-sticky-notes" @@ -130,3 +179,77 @@ start_url = "https://www.wbmason.com/" item = "a pack of Post-it 3 x 3 in canary yellow sticky notes" cart_url = "https://www.wbmason.com/shopping-cart" cart_match = 'Post-it|sticky note' +tier = "hard" + +[[tasks]] +id = "adidas-socks" +site = "adidas.com" +start_url = "https://www.adidas.com/us" +item = "one pack of adidas crew socks" +cart_url = "https://www.adidas.com/us/cart" +cart_match = 'sock' +tier = "hard" + +[[tasks]] +id = "bestbuy-hdmi-cable" +site = "bestbuy.com" +start_url = "https://www.bestbuy.com/" +item = "an Insignia HDMI cable that can be shipped" +cart_url = "https://www.bestbuy.com/cart" +cart_match = 'Insignia.{0,120}HDMI|HDMI.{0,120}Insignia' +tier = "hard" + +[[tasks]] +id = "sephora-niacinamide-serum" +site = "sephora.com" +start_url = "https://www.sephora.com/" +item = "one The Ordinary Niacinamide 10% + Zinc 1% serum that can be shipped" +cart_url = "https://www.sephora.com/basket" +# Specific enough not to match the basket page's promotions and recommendations. +cart_match = 'Niacinamide 10%\s*\+\s*Zinc' +tier = "hard" + +[[tasks]] +id = "nike-socks" +site = "nike.com" +start_url = "https://www.nike.com/" +item = "one pack of Nike Everyday crew socks" +cart_url = "https://www.nike.com/cart" +cart_match = 'Everyday|sock' +tier = "hard" + +[[tasks]] +id = "footlocker-socks" +site = "footlocker.com" +start_url = "https://www.footlocker.com/" +item = "one pack of crew socks" +cart_url = "https://www.footlocker.com/cart" +cart_match = 'sock' +tier = "hard" + +[[tasks]] +id = "chewy-dental-treats" +site = "chewy.com" +start_url = "https://www.chewy.com/" +item = "a bag or box of Greenies dental dog treats" +cart_url = "https://www.chewy.com/app/cart" +cart_match = 'Greenies' +tier = "hard" + +[[tasks]] +id = "bhphoto-microsd" +site = "bhphotovideo.com" +start_url = "https://www.bhphotovideo.com/" +item = "a SanDisk microSD memory card" +cart_url = "https://www.bhphotovideo.com/find/cart.jsp" +cart_match = 'SanDisk' +tier = "hard" + +[[tasks]] +id = "kohls-bath-towel" +site = "kohls.com" +start_url = "https://www.kohls.com/" +item = "one bath towel that can be shipped" +cart_url = "https://www.kohls.com/checkout/shopping_cart.jsp" +cart_match = 'towel' +tier = "hard" diff --git a/tests/test_definitions.py b/tests/test_definitions.py index 2ca3d28..2b672c6 100644 --- a/tests/test_definitions.py +++ b/tests/test_definitions.py @@ -1,3 +1,6 @@ +import pytest + +from harness import definitions, providers from harness.definitions import ROOT, load_arm, load_benchmark @@ -10,3 +13,22 @@ def test_every_benchmark_and_arm_resolves(): assert (ROOT / "toolsets" / arm.toolset / "toolset.py").is_file() if arm.browser.configurator: assert (ROOT / "configurators" / arm.browser.configurator / "configure").is_file() + + +def test_provider_arms_run_on_remote_browsers(): + benchmark = load_benchmark("retail-cart-providers") + for name in benchmark.arms: + arm = load_arm(name) + assert arm.environment in ("remote-browser", "kernel-vm-remote-browser") + assert arm.toolset == "playwright-mcp" + assert arm.browser.config["provider"] in providers.PROVIDERS + + +def test_unknown_environment_is_refused(tmp_path, monkeypatch): + (tmp_path / "arms").mkdir() + (tmp_path / "arms/bad.toml").write_text( + 'agent = "codex"\nmodel = "m"\ntoolset = "t"\nenvironment = "elsewhere"\n' + ) + monkeypatch.setattr(definitions, "ROOT", tmp_path) + with pytest.raises(SystemExit, match="elsewhere"): + load_arm("bad") diff --git a/tests/test_environment.py b/tests/test_environment.py index 2f282eb..4bbf392 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -1,5 +1,6 @@ import asyncio import json +import logging import pytest from harbor.environments.kernel import KernelEnvironment @@ -195,3 +196,92 @@ def test_vm_setup_commands_run_as_root_after_the_browser_is_created(tmp_path, mo asyncio.run(env._create_browser()) assert calls == [("exec", ["-c", "echo hi"], True), ("exec", ["-c", "true"], True)] + + +class FakeProvider: + env = () + + def __init__(self, fail=False): + self.fail = fail + self.calls = [] + + async def create(self, params): + self.calls.append(("create", params)) + if self.fail: + raise RuntimeError("no capacity") + return environment.providers.Session("remote-1", "wss://remote/1") + + async def release(self, session_id): + self.calls.append(("release", session_id)) + + +def _remote_environment(tmp_path, monkeypatch, provider, cls=None): + from harbor.environments.base import ExecResult + + cls = cls or environment.RemoteBrowserEnvironment + # The Harbor environment class the remote browser is layered on. + base = next(base for base in cls.__mro__[1:] if base.__module__.startswith("harbor.")) + + calls = [] + + async def start(self, force_build): + calls.append("container started") + + async def stop(self, delete): + calls.append("container stopped") + + async def exec(self, command, env=None, user=None, **kwargs): + calls.append((command, env, user)) + return ExecResult(return_code=0) + + def init(self, environment_dir, **kwargs): + self._persistent_env = {} + self.logger = logging.getLogger("test") + + monkeypatch.setattr(base, "__init__", init) + monkeypatch.setattr(base, "start", start) + monkeypatch.setattr(base, "stop", stop) + monkeypatch.setattr(base, "exec", exec) + monkeypatch.setitem(environment.providers.PROVIDERS, "fake", provider) + (tmp_path / "kernel-config.json").write_text( + json.dumps({"provider": "fake", "params": {"stealth": True}}) + ) + return cls(environment_dir=tmp_path), calls + + +def test_remote_browser_is_created_after_the_container_and_released_after(tmp_path, monkeypatch): + provider = FakeProvider() + env, calls = _remote_environment(tmp_path, monkeypatch, provider) + + asyncio.run(env.start(force_build=False)) + asyncio.run(env.stop(delete=True)) + + assert calls[0] == "container started" + command, exec_env, user = calls[1] + assert environment.CDP_URL_FILE in command + assert exec_env == {"CDP_URL": "wss://remote/1"} + assert user == "root" + assert env._persistent_env["BROWSER_SESSION_ID"] == "remote-1" + assert calls[2] == "container stopped" + assert provider.calls == [("create", {"stealth": True}), ("release", "remote-1")] + + +def test_remote_browser_creation_failure_is_identifiable(tmp_path, monkeypatch): + env, _ = _remote_environment(tmp_path, monkeypatch, FakeProvider(fail=True)) + with pytest.raises(environment.BrowserCreationError, match="no capacity"): + asyncio.run(env.start(force_build=False)) + + +def test_kernel_vm_remote_browser_uses_the_vm_only_as_compute(tmp_path, monkeypatch): + provider = FakeProvider() + env, calls = _remote_environment( + tmp_path, monkeypatch, provider, environment.KernelVmRemoteBrowserEnvironment + ) + + asyncio.run(env.start(force_build=False)) + asyncio.run(env.stop(delete=True)) + + assert calls[0] == "container started" + assert environment.CDP_URL_FILE in calls[1][0] and calls[1][1] == {"CDP_URL": "wss://remote/1"} + assert calls[2] == "container stopped" + assert provider.calls == [("create", {"stealth": True}), ("release", "remote-1")] diff --git a/tests/test_pinned.py b/tests/test_pinned.py index 157ee6f..9288471 100644 --- a/tests/test_pinned.py +++ b/tests/test_pinned.py @@ -14,12 +14,21 @@ def _configure(site): return json.loads(result.stdout) -def test_every_retail_cart_site_has_a_pinned_configuration(): +def test_measured_sites_are_pinned_and_new_sites_fall_back(): + import tomllib + from harness.definitions import load_suite + measured = tomllib.loads((CONFIGURE.parent / "sites.toml").read_text())["sites"] for task in load_suite("retail-cart").load_tasks(): output = _configure(task.site) - assert output["browser"]["proxy_create"]["config"] == {"country": "US"} + if task.site in measured: + assert output["browser"]["proxy_create"]["config"] == {"country": "US"} + else: + # Not yet measured by hand: the arm's static config, recorded as unpinned. + assert output["browser"] == {} and output["provenance"]["pinned"] is False + # The original 16 sites were all measured. + assert len(measured) >= 16 def test_personas_become_chromium_env(): diff --git a/tests/test_playwright_mcp.py b/tests/test_playwright_mcp.py new file mode 100644 index 0000000..35e8aaa --- /dev/null +++ b/tests/test_playwright_mcp.py @@ -0,0 +1,20 @@ +import tomllib + +from harness.definitions import load_toolset + + +def test_apply_adds_the_server_install_and_instruction(task_dir): + load_toolset("playwright-mcp").apply([task_dir]) + + environment = task_dir / "environment" + assert (environment / "playwright-mcp/bin/browser-mcp").is_file() + assert "npm install --global @playwright/mcp@0.0.84" in (environment / "Dockerfile").read_text() + config = tomllib.loads((task_dir / "task.toml").read_text()) + assert config["environment"]["mcp_servers"][-1]["command"] == "/usr/local/bin/browser-mcp" + assert config["metadata"]["playwright_mcp_version"] == "0.0.84" + assert "`browser` MCP server" in (task_dir / "steps/run/instruction.md").read_text() + + +def test_ref_picks_the_playwright_mcp_version(task_dir): + load_toolset("playwright-mcp").apply([task_dir], ref="0.0.80") + assert "@playwright/mcp@0.0.80" in (task_dir / "environment/Dockerfile").read_text() diff --git a/tests/test_providers.py b/tests/test_providers.py new file mode 100644 index 0000000..ce41973 --- /dev/null +++ b/tests/test_providers.py @@ -0,0 +1,145 @@ +import asyncio +import json + +import httpx +import pytest + +from harness import providers + + +def _serve(monkeypatch, provider, handler): + requests = [] + + def record(request): + requests.append(request) + return handler(request) + + monkeypatch.setattr( + provider, + "_client", + lambda: httpx.AsyncClient( + base_url=provider.base_url, + headers={provider.key_header: "key"}, + transport=httpx.MockTransport(record), + ), + ) + return requests + + +def test_browserbase_sends_the_project_and_releases_the_session(monkeypatch): + monkeypatch.setenv("BROWSERBASE_PROJECT_ID", "proj") + provider = providers.get("browserbase") + requests = _serve( + monkeypatch, + provider, + lambda request: httpx.Response(200, json={"id": "bb1", "connectUrl": "wss://bb/1"}), + ) + + session = asyncio.run(provider.create({"keepAlive": True})) + asyncio.run(provider.release(session.id)) + + assert session == providers.Session("bb1", "wss://bb/1") + assert json.loads(requests[0].content) == {"projectId": "proj", "keepAlive": True} + assert requests[1].url.path == "/v1/sessions/bb1" + assert json.loads(requests[1].content)["status"] == "REQUEST_RELEASE" + + +def test_steel_connects_with_the_session_and_key(monkeypatch): + monkeypatch.setenv("STEEL_API_KEY", "sk") + provider = providers.get("steel") + requests = _serve(monkeypatch, provider, lambda request: httpx.Response(200, json={"id": "s1"})) + + session = asyncio.run(provider.create({"useProxy": True})) + asyncio.run(provider.release("s1")) + + assert session.cdp_url == "wss://connect.steel.dev?apiKey=sk&sessionId=s1" + assert [request.url.path for request in requests] == ["/v1/sessions", "/v1/sessions/s1/release"] + + +def test_steel_names_the_project_for_an_organization_key(monkeypatch): + monkeypatch.setenv("STEEL_API_KEY", "sk") + monkeypatch.setenv("STEEL_PROJECT_ID", "sp") + provider = providers.get("steel") + requests = _serve(monkeypatch, provider, lambda request: httpx.Response(200, json={"id": "s1"})) + + asyncio.run(provider.create({"useProxy": True})) + asyncio.run(provider.release("s1")) + + assert json.loads(requests[0].content) == {"projectId": "sp", "useProxy": True} + assert json.loads(requests[1].content) == {"projectId": "sp"} + + +def test_browser_use_creates_and_stops_a_browser(monkeypatch): + provider = providers.get("browser-use") + requests = _serve( + monkeypatch, + provider, + lambda request: httpx.Response(200, json={"id": "bu1", "cdpUrl": "wss://bu/1"}), + ) + + session = asyncio.run(provider.create({"proxyCountryCode": "us"})) + asyncio.run(provider.release(session.id)) + + assert session == providers.Session("bu1", "wss://bu/1") + assert json.loads(requests[0].content) == {"proxyCountryCode": "us"} + assert requests[1].method == "PATCH" and requests[1].url.path == "/api/v4/browsers/bu1" + assert json.loads(requests[1].content) == {"action": "stop"} + + +def test_kernel_sends_params_the_sdk_does_not_know_in_the_body(monkeypatch): + calls = [] + + class Browsers: + async def create(self, *, stealth=None, timeout_seconds=None, extra_body=None): + calls.append({"stealth": stealth, "extra_body": extra_body}) + return type("Browser", (), {"session_id": "k1", "cdp_ws_url": "wss://k/1"}) + + class Client: + browsers = Browsers() + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return None + + monkeypatch.setattr(providers, "AsyncKernel", Client) + session = asyncio.run( + providers.get("kernel").create({"stealth": True, "fingerprint": {"os": "macos"}}) + ) + + assert session == providers.Session("k1", "wss://k/1") + assert calls == [{"stealth": True, "extra_body": {"fingerprint": {"os": "macos"}}}] + + +def test_hyperbrowser_and_anchor_read_their_cdp_urls(monkeypatch): + hyperbrowser = providers.get("hyperbrowser") + _serve( + monkeypatch, + hyperbrowser, + lambda request: httpx.Response(200, json={"id": "h1", "wsEndpoint": "wss://hb/1"}), + ) + anchor = providers.get("anchor") + _serve( + monkeypatch, + anchor, + lambda request: httpx.Response(200, json={"data": {"id": "a1", "cdp_url": "wss://a/1"}}), + ) + + # Hyperbrowser stops a session on CDP disconnect unless the URL keeps it alive. + assert asyncio.run(hyperbrowser.create({})) == providers.Session( + "h1", "wss://hb/1?keepAlive=true" + ) + assert asyncio.run(anchor.create({})) == providers.Session("a1", "wss://a/1") + + +def test_failed_create_raises(monkeypatch): + provider = providers.get("hyperbrowser") + _serve(monkeypatch, provider, lambda request: httpx.Response(402, json={})) + with pytest.raises(httpx.HTTPStatusError): + asyncio.run(provider.create({})) + + +def test_unknown_provider_is_refused(): + with pytest.raises(SystemExit, match="nope"): + providers.get("nope") diff --git a/tests/test_retail_cart.py b/tests/test_retail_cart.py index 962c9a5..37544dc 100644 --- a/tests/test_retail_cart.py +++ b/tests/test_retail_cart.py @@ -32,6 +32,19 @@ def test_generate_writes_a_harbor_task(tmp_path): assert (task_dir / "environment/bench/finalize-mcp").is_file() +def test_tiers_filter_tasks_and_are_recorded(tmp_path): + tasks = suite.load_tasks() + assert {task.tier for task in tasks} == set(suite.TIERS) + hard = [task.id for task in tasks if task.tier == "hard"] + task_dirs = suite.generate(None, tmp_path, tiers="hard") + assert sorted(path.name for path in task_dirs) == sorted(hard) + config = tomllib.loads((task_dirs[0] / "task.toml").read_text()) + assert config["metadata"]["tier"] == "hard" + assert len(suite.generate(None, tmp_path / "all")) == len(tasks) + with pytest.raises(SystemExit, match="Unknown tiers: extreme"): + suite.generate(None, tmp_path, tiers=["extreme"]) + + def test_unknown_task_is_refused(tmp_path): with pytest.raises(SystemExit, match="nope"): suite.generate(["nope"], tmp_path) diff --git a/tests/test_run.py b/tests/test_run.py index e74d2bb..e91909a 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -67,3 +67,9 @@ def test_harbor_command_passes_agent_kwargs_as_json(tmp_path): assert "version=0.163.0" in command assert "disable_web_search=true" in command assert command[command.index("--n-attempts") + 1] == "3" + + +def test_harbor_command_picks_the_arms_environment(tmp_path): + arm = Arm(name="a", agent="codex", model="m", toolset="t", environment="remote-browser") + command = run.harbor_command(run.PreparedArm(arm, tmp_path, {}), tmp_path, "job", 1) + assert command[command.index("--env") + 1] == "harness.environment:RemoteBrowserEnvironment" diff --git a/toolsets/playwright-mcp/image/bin/browser-mcp b/toolsets/playwright-mcp/image/bin/browser-mcp new file mode 100755 index 0000000..8063fa4 --- /dev/null +++ b/toolsets/playwright-mcp/image/bin/browser-mcp @@ -0,0 +1,6 @@ +#!/bin/sh +# Playwright MCP attached over CDP to the trial's remote browser, whose URL the +# environment writes to a file before the agent starts. +set -eu +exec playwright-mcp --cdp-endpoint "$(cat /run/browser/cdp-url)" \ + --cdp-timeout 60000 --output-dir /tmp/playwright-mcp "$@" diff --git a/toolsets/playwright-mcp/image/layer.Dockerfile b/toolsets/playwright-mcp/image/layer.Dockerfile new file mode 100644 index 0000000..d3753c1 --- /dev/null +++ b/toolsets/playwright-mcp/image/layer.Dockerfile @@ -0,0 +1,3 @@ +# playwright-mcp toolset. Appended to the suite's Dockerfile, which must have +# Node.js. The toolset appends the line installing its @playwright/mcp version. +COPY playwright-mcp/bin/ /usr/local/bin/ diff --git a/toolsets/playwright-mcp/toolset.py b/toolsets/playwright-mcp/toolset.py new file mode 100644 index 0000000..c6639fe --- /dev/null +++ b/toolsets/playwright-mcp/toolset.py @@ -0,0 +1,61 @@ +"""Playwright MCP, attached over CDP to a browser from any provider. + +The same tools drive every provider's browser, so arms that differ only in +provider compare the browsers rather than the tools. Needs an arm with +`environment = "remote-browser"`, which writes the browser's CDP URL into the +task container. The agent works in the browser's existing page and context. + +Toolset ref: the @playwright/mcp version to install, default `VERSION`. +""" + +from __future__ import annotations + +import shutil +from pathlib import Path + +import tomlkit + +from harness.tasks import append_instruction, read_config, write_config + +VERSION = "0.0.84" +IMAGE = Path(__file__).with_name("image") + +INSTRUCTION = """ +## Browser tools + +- Use the `browser` MCP server's tools for all browser interaction. They drive a browser that is already open on the start page; keep working in it. +- Do not install or launch another browser. +""" + + +def apply(task_dirs: list[Path], ref: str | None = None) -> None: + for task_dir in task_dirs: + _apply(task_dir, ref or VERSION) + + +def _apply(task_dir: Path, version: str) -> None: + context = task_dir / "environment" / "playwright-mcp" + shutil.copytree(IMAGE / "bin", context / "bin") + dockerfile = task_dir / "environment" / "Dockerfile" + dockerfile.write_text( + dockerfile.read_text().rstrip() + + "\n\n" + + (IMAGE / "layer.Dockerfile").read_text() + + f"RUN npm install --global @playwright/mcp@{version}\n" + ) + + config = read_config(task_dir) + server = tomlkit.table() + server.update( + { + "name": "browser", + "transport": "stdio", + "command": "/usr/local/bin/browser-mcp", + "args": [], + } + ) + servers = config["environment"].setdefault("mcp_servers", tomlkit.aot()) + servers.append(server) + config.setdefault("metadata", tomlkit.table())["playwright_mcp_version"] = version + write_config(task_dir, config) + append_instruction(task_dir, INSTRUCTION)