diff --git a/CHANGELOG.md b/CHANGELOG.md index 8803ac90..00116771 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] ### Added - Added Kernel as a managed remote browser runtime with live view and downloaded replay recordings. Thanks to @[rgarcia](https://github.com/rgarcia). +- Added a `--browser-runtime kernel` mode to the Harbor adapter that runs each task against one Kernel cloud browser, exposing only a credential-free CDP bridge to the agent, and finalizes the replay and deletes the browser during verification. ## [0.9.2] - 2026-08-18 ### Added diff --git a/docs/harbor.md b/docs/harbor.md index 9a0453e2..0dbba906 100644 --- a/docs/harbor.md +++ b/docs/harbor.md @@ -45,7 +45,7 @@ uv run clawbench-harbor-adapt \ ## 2. Wire up the judge -Harbor's verifier calls the same judge ClawBench uses. Export the four variables once, then forward them into each run with `--ve`: +Harbor's verifier applies both ClawBench V2 judge rubrics to every intercepted request. `reward` and `reward_lenient` use the public leaderboard's no-explicit-contradiction rubric; `reward_strict` requires the payload to demonstrate complete fulfillment. The two judge calls run concurrently against the same request and use the model configured below. Export the four variables once, then forward them into each run with `--ve`: ```bash export CLAWBENCH_JUDGE_BASE_URL="https://your-judge-provider.example/v1" @@ -108,6 +108,25 @@ uvx --from harbor==0.15.0 harbor run \ --jobs-dir ./harbor-jobs/hermes-deepseek-flash ``` +## Kernel browser runtime (control arm) + +By default each Harbor trial runs Chromium inside its own container. Pass `--browser-runtime kernel` to the adapter to run the same tasks against one Kernel cloud browser per task instead: + +```bash +uv run clawbench-harbor-adapt \ + --output-dir ./harbor-datasets/clawbench-v2-kernel \ + --browser-runtime kernel \ + --browser-runtime-options '{"stealth": true}' \ + --task-ids v2-1134-chapter-finder-redcross \ + --overwrite +``` + +During task setup, the environment creates exactly one Kernel browser and replay, starts the ClawBench runtime server against it, and exposes only the local credential-free CDP bridge (`http://127.0.0.1:7878`) to the agent — the Kernel API key is never visible to the benchmark agent. Session identity and cleanup metadata land in `/my-info/kernel_browser.json`. During verification the provider replay is finalized, `recording.mp4` is downloaded into `/data`, and the browser is deleted (idempotently, including failure paths via a setup trap). + +Generated tasks register a pinned Playwright MCP package (`@playwright/mcp@0.0.79`) pointed at the CDP bridge, so Harbor's stock Claude Code and Codex agents drive the Kernel browser with native Playwright MCP tool calls — structurally identical to ClawBench's native Claude/Codex harnesses. + +Export `KERNEL_API_KEY` (and optionally `KERNEL_BASE_URL` for non-production gateways) before `harbor run`; no extra flags are needed. + ## Making it fast A full V2 sweep is 129 containerized browser sessions, each capped by the task's `time_limit`. Serial, that is a very long night. What actually moves the needle, in order: @@ -140,7 +159,7 @@ uvx --from harbor==0.15.0 harbor run -p ./harbor-datasets/clawbench-v2-smoke \ Each converted task directory carries its own `environment/` (Chromium, the ClawBench recorder/interceptor, noVNC, runtime helper scripts), a `run/` step with `instruction.md`, the original `task.json`, the `eval-schema.json`, and a verifier under `tests/`. It deliberately contains **no ClawBench-native harness** — Harbor installs and runs whatever agent you pass to `-a` inside the task container. -Scoring is the same two-stage rule as the native runner: the interceptor must catch a request matching the task schema, and the judge must agree the payload fulfills the instruction. +Scoring uses the same two-stage rule as the native runner: the interceptor must catch a request matching the task schema, then the verifier emits both the public lenient reward and the conservative strict reward. Harbor uses the lenient result as the primary `reward` metric and retains both verdicts and reasons in `clawbench-result.json`. ## Troubleshooting diff --git a/src/clawbench/eval/harbor_adapter.py b/src/clawbench/eval/harbor_adapter.py index 4639bdce..2f95db18 100644 --- a/src/clawbench/eval/harbor_adapter.py +++ b/src/clawbench/eval/harbor_adapter.py @@ -11,11 +11,24 @@ from pathlib import Path from typing import Any +from clawbench.runner.run_support.browser_runtime.providers import ( + BrowserRuntimeError, + _parse_options, +) from clawbench.runner.run_support.task import build_instruction, validate_task_data from clawbench.utils.paths import RUNTIME_ROOT, asset_path DEFAULT_CASES_DIR = asset_path("test-cases", "v2") STEP_NAME = "run" +HARBOR_BROWSER_RUNTIMES = ("local", "kernel") +# Remote browser runtimes connect through the runtime server's local, +# credential-free CDP bridge instead of a container-local Chromium. +REMOTE_BRIDGE_CDP_URL = "http://127.0.0.1:7878" +LOCAL_CDP_URL = "http://127.0.0.1:9223" +# Pinned Playwright MCP package Harbor's stock Claude Code and Codex agents +# use to drive the ClawBench browser through the CDP bridge. +PLAYWRIGHT_MCP_PACKAGE = "@playwright/mcp" +PLAYWRIGHT_MCP_VERSION = "0.0.79" def sanitize_task_name(raw: str) -> str: @@ -96,6 +109,16 @@ def copy_environment(env_dir: Path) -> None: copytree_filtered(RUNTIME_ROOT / "shared", env_dir / "shared") copytree_filtered(RUNTIME_ROOT / "harbor", env_dir / "harbor") (env_dir / "harbor" / "Dockerfile").unlink(missing_ok=True) + # The Kernel lifecycle scripts reuse the same provider implementation as + # the native runner. + shutil.copy2( + Path(__file__).resolve().parents[1] + / "runner" + / "run_support" + / "browser_runtime" + / "providers.py", + env_dir / "harbor" / "browser_runtime_providers.py", + ) shutil.copy2( Path(__file__).resolve().parents[1] / "runner" @@ -108,18 +131,71 @@ def copy_environment(env_dir: Path) -> None: chmod_executable(script) -def harbor_instruction(task: dict[str, Any]) -> str: +def playwright_mcp_server(cdp_url: str) -> dict[str, Any]: + return { + "name": "playwright", + "transport": "stdio", + "command": "npx", + "args": [ + "-y", + f"{PLAYWRIGHT_MCP_PACKAGE}@{PLAYWRIGHT_MCP_VERSION}", + "--cdp-endpoint", + cdp_url, + ], + } + + +def mcp_servers_toml(servers: list[dict[str, Any]]) -> str: + if not servers: + return "" + blocks = [] + for server in servers: + args = ", ".join(json.dumps(arg) for arg in server["args"]) + blocks.append( + "[[environment.mcp_servers]]\n" + f"name = {json.dumps(server['name'])}\n" + f"transport = {json.dumps(server['transport'])}\n" + f"command = {json.dumps(server['command'])}\n" + f"args = [{args}]\n" + ) + return "\n" + "\n".join(blocks) + + +def harbor_instruction(task: dict[str, Any], *, browser_runtime: str = "local") -> str: instruction = build_instruction(task) - return ( - instruction + "\n\n---\n" + cdp_url = REMOTE_BRIDGE_CDP_URL if browser_runtime == "kernel" else LOCAL_CDP_URL + runtime_section = ( "Harbor browser runtime:\n" - "- Use the existing Chromium session exposed by Chrome DevTools Protocol.\n" - "- CDP endpoint: http://127.0.0.1:9223\n" + "- Use the existing browser session exposed by Chrome DevTools Protocol.\n" + f"- CDP endpoint: {cdp_url}\n" "- CDP environment variables are also set for the agent process: " "CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL.\n" - "- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html\n" - "- Do not launch a separate browser. Complete the task through the existing browser session.\n" - "---\n" + ) + if browser_runtime == "local": + runtime_section += "- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html\n" + restrictions = ( + "Task constraints (matching the native ClawBench harness rules):\n" + f"- Time limit: {task.get('time_limit')} minutes. The harness stops the " + "run when the limit elapses, so finish and submit before then.\n" + "- Complete the task entirely in the browser; do not launch or use any " + "other browser.\n" + "- Use only Playwright MCP browser tools plus reading files under " + "./my-info/ to accomplish the task.\n" + "- Do NOT make direct HTTP/network requests for task completion via shell " + "tools, scripts, API calls, or SMTP — every task action must go through " + "the browser.\n" + "- Submit through the browser: perform the task's final submission action " + "in the browser so the request happens on the page.\n" + "- Stop after submission: once the final action is submitted, stop and do " + "not start other work.\n" + ) + return ( + instruction + + "\n\n---\n" + + runtime_section + + "- Do not launch a separate browser. Complete the task through the existing browser session.\n" + + "---\n\n" + + restrictions ) @@ -130,12 +206,37 @@ def task_toml( dataset_name: str, timeout_sec: int, task_dir_name: str, + browser_runtime: str = "local", + browser_runtime_options: str | None = None, ) -> str: escaped_description = json.dumps(description) escaped_dataset = json.dumps(dataset_name) escaped_source = json.dumps(task_dir_name) escaped_package = json.dumps(package_name) - return f"""schema_version = "1.3" + cdp_url = REMOTE_BRIDGE_CDP_URL if browser_runtime == "kernel" else LOCAL_CDP_URL + kernel_env = "" + mcp_servers = "" + if browser_runtime == "kernel": + runtime_options_line = ( + f"\nCLAWBENCH_BROWSER_RUNTIME_OPTIONS = {json.dumps(browser_runtime_options)}" + if browser_runtime_options + else '\nCLAWBENCH_BROWSER_RUNTIME_OPTIONS = "${CLAWBENCH_BROWSER_RUNTIME_OPTIONS:-}"' + ) + kernel_env = ( + '\nCLAWBENCH_HARBOR_BROWSER_RUNTIME = "kernel"' + '\nKERNEL_API_KEY = "${KERNEL_API_KEY}"' + '\nKERNEL_BASE_URL = "${KERNEL_BASE_URL:-}"' + + runtime_options_line + + '\nCLAWBENCH_RECORDING_MODE = "provider-download"' + ) + mcp_servers = mcp_servers_toml([playwright_mcp_server(REMOTE_BRIDGE_CDP_URL)]) + healthcheck_command = ( + "curl -sf http://127.0.0.1:7878/api/status | grep -q '" + + '\\"eval_interceptor_ready\\":true' + + f"' && curl -sf {cdp_url}/json/version >/dev/null" + ) + return ( + f"""schema_version = "1.3" source = "clawbench-v2" artifacts = ["/data"] @@ -147,20 +248,24 @@ def task_toml( [metadata] dataset = {escaped_dataset} source_task = {escaped_source} +browser_runtime = "{browser_runtime}" [environment] build_timeout_sec = 1200.0 network_mode = "public" -workdir = "/app" +# The container root doubles as the step workdir so every exec runs with a +# cwd that exists even on overlay drivers that cannot resolve image-created +# directories during `docker exec`. +workdir = "/" [environment.env] PURELY_MAIL_API_KEY = "${{PURELY_MAIL_API_KEY}}" PURELY_MAIL_DOMAIN = "${{PURELY_MAIL_DOMAIN}}" -CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" -BROWSER_CDP_URL = "http://127.0.0.1:9223" -CDP_URL = "http://127.0.0.1:9223" -CHROME_CDP_URL = "http://127.0.0.1:9223" -PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_CDP_URL = "{cdp_url}" +BROWSER_CDP_URL = "{cdp_url}" +CDP_URL = "{cdp_url}" +CHROME_CDP_URL = "{cdp_url}" +PLAYWRIGHT_CDP_URL = "{cdp_url}"{kernel_env} CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" CLAWBENCH_JUDGE_BASE_URL = "${{CLAWBENCH_JUDGE_BASE_URL:-}}" @@ -175,36 +280,62 @@ def task_toml( timeout_sec = {float(timeout_sec):.1f} [steps.verifier] -timeout_sec = 180.0 +timeout_sec = 300.0 [steps.healthcheck] -command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\\\"eval_interceptor_ready\\\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +command = "{healthcheck_command}" interval_sec = 2.0 timeout_sec = 5.0 start_period_sec = 2.0 start_interval_sec = 1.0 retries = 30 """ + + mcp_servers + ) -def setup_script() -> str: - return """#!/bin/bash +def setup_script(browser_runtime: str = "local") -> str: + kernel_setup = "" + readiness = ( + " if curl -sf http://127.0.0.1:7878/api/status >/dev/null \\\n" + " && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then\n" + ) + if browser_runtime == "kernel": + readiness = ( + " if curl -sf http://127.0.0.1:7878/api/status >/dev/null \\\n" + " && curl -sf http://127.0.0.1:7878/json/version >/dev/null; then\n" + ) + kernel_setup = ( + "# Create the Kernel browser and replay before the runtime server" + " starts so it can bridge the provider CDP endpoint.\n" + "/app/src/runtime-server/.venv/bin/python /app/src/harbor/kernel-browser.py start\n" + "export CLAWBENCH_BROWSER_CDP_URL_FILE=/tmp/clawbench-run/kernel-cdp-url\n" + "\n" + "cleanup_browser() {\n" + " /app/src/runtime-server/.venv/bin/python /app/src/harbor/kernel-browser.py cleanup || true\n" + "}\n" + "trap cleanup_browser EXIT\n" + "\n" + ) + return f"""#!/bin/bash set -euo pipefail -mkdir -p /data /logs/verifier /app/extra_info -cp /app/eval-schema.json /eval-schema.json +mkdir -p /data /logs/verifier /extra_info /app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py \ - --task-json /app/task.json \ - --extra-info-dir /app/extra_info \ - --output-dir /app/my-info + --task-json /task.json \ + --extra-info-dir /extra_info \ + --output-dir /my-info + +# Harbor installs its stock agent before step setup. Wrap that executable so +# ClawBench's existing /data/.stop-requested signal ends the agent cleanly. +/app/src/harbor/wrap-harbor-agent.sh -/app/src/harbor/start-runtime.sh +{kernel_setup}/app/src/harbor/start-runtime.sh for _ in $(seq 1 60); do - if curl -sf http://127.0.0.1:7878/api/status >/dev/null \ - && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then - rm -f /app/setup.sh +{readiness} rm -f /app/setup.sh + trap - EXIT exit 0 fi sleep 1 @@ -215,15 +346,21 @@ def setup_script() -> str: """ -def test_script() -> str: - return """#!/bin/bash +def test_script(browser_runtime: str = "local") -> str: + kernel_finalize = "" + if browser_runtime == "kernel": + kernel_finalize = ( + "# Stop the provider replay, download the recording, delete the browser.\n" + "/app/src/runtime-server/.venv/bin/python /app/src/harbor/kernel-browser.py finalize\n" + ) + return f"""#!/bin/bash set -euo pipefail curl -sf -X POST http://127.0.0.1:7878/api/stop || true curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true sleep 2 rm -f /data/.stop-requested -rm -rf /logs/verifier/data +{kernel_finalize}rm -rf /logs/verifier/data cp -a /data /logs/verifier/data /app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py @@ -265,6 +402,8 @@ def write_harbor_task( output_name: str, org: str, dataset_name: str, + browser_runtime: str = "local", + browser_runtime_options: str | None = None, ) -> Path: dest = output_root / output_name if dest.exists(): @@ -292,15 +431,19 @@ def write_harbor_task( dataset_name=dataset_name, timeout_sec=timeout_sec, task_dir_name=task_dir.name, + browser_runtime=browser_runtime, + browser_runtime_options=browser_runtime_options, ) ) - (step_dir / "instruction.md").write_text(harbor_instruction(task)) + (step_dir / "instruction.md").write_text( + harbor_instruction(task, browser_runtime=browser_runtime) + ) (workdir / "eval-schema.json").write_text(json.dumps(task["eval_schema"], indent=2)) (workdir / "task.json").write_text(json.dumps(task, indent=2, ensure_ascii=False)) copy_extra_info(task, task_dir, workdir / "extra_info") - write_text_executable(workdir / "setup.sh", setup_script()) + write_text_executable(workdir / "setup.sh", setup_script(browser_runtime)) (tests_dir / "task.json").write_text(json.dumps(task, indent=2, ensure_ascii=False)) - write_text_executable(tests_dir / "test.sh", test_script()) + write_text_executable(tests_dir / "test.sh", test_script(browser_runtime)) write_text_executable(solution_dir / "solve.sh", solve_script()) copy_environment(env_dir) return dest @@ -339,6 +482,18 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Overwrite an existing output directory", ) + parser.add_argument( + "--browser-runtime", + choices=HARBOR_BROWSER_RUNTIMES, + default="local", + help="Browser runtime for the generated tasks; kernel creates one " + "Kernel browser per task during Harbor setup", + ) + parser.add_argument( + "--browser-runtime-options", + default=None, + help="JSON object with runtime options, e.g. '{\"stealth\":true}'", + ) return parser @@ -346,6 +501,13 @@ def main(argv: list[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(argv) + options: dict[str, Any] = {} + if args.browser_runtime_options: + try: + options = _parse_options(args.browser_runtime_options) + except BrowserRuntimeError as exc: + parser.error(str(exc)) + cases_dir = (args.cases_dir or DEFAULT_CASES_DIR).resolve() default_cases = args.cases_dir is None if not cases_dir.exists(): @@ -384,10 +546,16 @@ def main(argv: list[str] | None = None) -> int: output_name=out_name, org=args.org, dataset_name=args.dataset_name, + browser_runtime=args.browser_runtime, + browser_runtime_options=(json.dumps(options) if options else None), ) ) print(f"Wrote {len(written)} Harbor task(s) to {output_dir}") + if args.browser_runtime != "local": + print(f"Browser runtime: {args.browser_runtime}") + if options: + print(f"Runtime options: {json.dumps(options)}") return 0 diff --git a/src/clawbench/runner/run_support/browser_runtime/providers.py b/src/clawbench/runner/run_support/browser_runtime/providers.py index 999da3a7..9a852082 100644 --- a/src/clawbench/runner/run_support/browser_runtime/providers.py +++ b/src/clawbench/runner/run_support/browser_runtime/providers.py @@ -558,6 +558,16 @@ def _delete(self, session_id: str) -> str: raise return "deleted" + def session_exists(self, session_id: str) -> bool: + """Return True if the provider still reports this browser session.""" + try: + self._request("GET", f"/browsers/{session_id}") + except _KernelApiError as e: + if e.status == 404: + return False + raise + return True + def _stop_replay(self, session_id: str, replay_id: str) -> None: try: self._request( @@ -575,13 +585,9 @@ def start(self, task: dict[str, Any], time_limit_s: int) -> BrowserSession: timeout_seconds = min(259200, max(10, time_limit_s + 120)) payload = { **self.options, + "stealth": self.options.get("stealth", True), "headless": False, "timeout_seconds": timeout_seconds, - "viewport": { - "width": 1920, - "height": 1080, - "refresh_rate": 25, - }, } try: result = self._request_json("POST", "/browsers", payload) diff --git a/src/clawbench/runtime/harbor/Dockerfile b/src/clawbench/runtime/harbor/Dockerfile index cd7c1921..d804a557 100644 --- a/src/clawbench/runtime/harbor/Dockerfile +++ b/src/clawbench/runtime/harbor/Dockerfile @@ -8,7 +8,7 @@ RUN echo 'APT::Sandbox::User "root";' > /etc/apt/apt.conf.d/01disable-sandbox \ && printf '#!/bin/sh\n/usr/bin/dpkg-statoverride.real "$@" 2>/dev/null || true\n' \ > /usr/bin/dpkg-statoverride && chmod +x /usr/bin/dpkg-statoverride \ && apt-get update && apt-get install -y --no-install-recommends \ - chromium xvfb ffmpeg socat curl git x11vnc xclip \ + chromium xvfb ffmpeg socat curl git x11vnc xclip nodejs npm \ libegl1 libgbm1 \ fonts-noto-color-emoji fonts-noto-cjk \ && mv /usr/bin/dpkg-statoverride.real /usr/bin/dpkg-statoverride \ diff --git a/src/clawbench/runtime/harbor/kernel-browser.py b/src/clawbench/runtime/harbor/kernel-browser.py new file mode 100644 index 00000000..7a346e51 --- /dev/null +++ b/src/clawbench/runtime/harbor/kernel-browser.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python3 +"""Kernel browser lifecycle for the ClawBench Harbor control arm. + +Mirrors the native runner's browser-runtime phases (start / finalize / +cleanup) inside a Harbor task container, reusing the same provider +implementation the native runner uses. The Kernel API key and the real +provider CDP URL stay in root-only files under /tmp/clawbench-run; the +benchmark agent only ever sees the credential-free CDP bridge exposed by +the ClawBench runtime server. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import sys +import time +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +try: # Vendored next to this script inside Harbor task environments. + from browser_runtime_providers import ( # type: ignore[import-not-found] + BrowserRuntimeError, + BrowserSession, + KernelRuntimeProvider, + redact_cdp_url, + ) +except ImportError: # Source checkout / tests. + from clawbench.runner.run_support.browser_runtime.providers import ( + BrowserRuntimeError, + BrowserSession, + KernelRuntimeProvider, + redact_cdp_url, + ) + +DATA_DIR = Path(os.environ.get("CLAWBENCH_DATA_DIR", "/data")) +STATE_DIR = Path("/tmp/clawbench-run") +STATE_FILE = STATE_DIR / "kernel-browser-state.json" +CDP_URL_FILE = STATE_DIR / "kernel-cdp-url" +METADATA_FILE = ( + Path(os.environ.get("CLAWBENCH_MY_INFO_DIR", "/my-info")) / "kernel_browser.json" +) +LIFECYCLE_FILE = DATA_DIR / "kernel-browser-lifecycle.json" +TASK_FILE = Path("/task.json") +RUNTIME_SERVER_BRIDGE_URL = "http://127.0.0.1:7878" + + +def _load_provider() -> KernelRuntimeProvider: + api_key = os.environ.get("KERNEL_API_KEY", "") + if not api_key: + raise SystemExit("KERNEL_API_KEY is required for the kernel browser runtime") + options: dict[str, Any] = {} + raw_options = os.environ.get("CLAWBENCH_BROWSER_RUNTIME_OPTIONS", "") + if raw_options: + options = json.loads(raw_options) + if not isinstance(options, dict): + raise SystemExit("CLAWBENCH_BROWSER_RUNTIME_OPTIONS must be a JSON object") + return KernelRuntimeProvider( + api_key=api_key, + options=options, + api_url=os.environ.get("KERNEL_BASE_URL") or "https://api.onkernel.com", + ) + + +def _load_state() -> dict[str, Any] | None: + if STATE_FILE.is_file(): + return json.loads(STATE_FILE.read_text()) + # Fall back to the agent-visible metadata so cleanup still works if the + # state file was lost but the browser was created. + if METADATA_FILE.is_file(): + metadata = json.loads(METADATA_FILE.read_text()) + if metadata.get("session_id"): + return {"metadata": metadata, "events": []} + return None + + +def _save_state(state: dict[str, Any]) -> None: + STATE_DIR.mkdir(parents=True, exist_ok=True) + STATE_FILE.write_text(json.dumps(state, indent=2)) + STATE_FILE.chmod(0o600) + if state.get("cdp_url"): + CDP_URL_FILE.write_text(str(state["cdp_url"])) + CDP_URL_FILE.chmod(0o600) + + +def _public_metadata(state: dict[str, Any]) -> dict[str, Any]: + """Credential-free view of the session for the agent-visible file.""" + metadata = state.get("metadata", {}) + inner = ( + metadata.get("metadata") if isinstance(metadata.get("metadata"), dict) else {} + ) + return { + "provider": "kernel", + "mode": "remote", + "runtime": "kernel", + "session_id": metadata.get("session_id"), + "replay_id": inner.get("replay_id") or metadata.get("replay_id"), + "region": inner.get("region") or metadata.get("region"), + "stealth": ( + inner["stealth"] if "stealth" in inner else metadata.get("stealth") + ), + "timeout_seconds": ( + inner.get("timeout_seconds") or metadata.get("timeout_seconds") + ), + "cdp_url": redact_cdp_url(metadata["cdp_url"]) + if metadata.get("cdp_url") + else None, + "viewer_url": "[REDACTED]" if metadata.get("viewer_url") else None, + "cdp_bridge_url": RUNTIME_SERVER_BRIDGE_URL, + "recording_mode": "provider-download", + "status": state.get("status"), + "cleanup_status": state.get("cleanup_status"), + "cleanup_error": state.get("cleanup_error"), + "deletion_verified": state.get("deletion_verified"), + "events": state.get("events", []), + } + + +def _write_metadata(state: dict[str, Any]) -> None: + METADATA_FILE.parent.mkdir(parents=True, exist_ok=True) + METADATA_FILE.write_text(json.dumps(_public_metadata(state), indent=2)) + + +def _record(state: dict[str, Any], event: str, **fields: Any) -> None: + entry = {"event": event, "ts": time.time(), **fields} + state.setdefault("events", []).append(entry) + print(json.dumps(entry), flush=True) + + +def _task_time_limit_s() -> int: + task = json.loads(TASK_FILE.read_text()) + return int(float(task["time_limit"]) * 60) + + +def cmd_start() -> int: + if STATE_FILE.is_file(): + state = json.loads(STATE_FILE.read_text()) + if state.get("status") == "created": + print("Kernel browser already created for this trial", flush=True) + _write_metadata(state) + return 0 + + provider = _load_provider() + task = json.loads(TASK_FILE.read_text()) + session = provider.start(task, _task_time_limit_s()) + state: dict[str, Any] = { + "status": "created", + "cdp_url": session.cdp_url, + "metadata": session.to_metadata(), + "events": [], + } + # to_metadata redacts the CDP URL; keep the real one for the runtime + # server only, in the 0600 state file. + state["metadata"]["cdp_url"] = session.cdp_url + _record(state, "browser_created", session_id=session.session_id) + _save_state(state) + _write_metadata(state) + print( + f"Kernel browser ready; CDP bridge at {RUNTIME_SERVER_BRIDGE_URL}", flush=True + ) + return 0 + + +def _download_recording(provider: KernelRuntimeProvider, state: dict[str, Any]) -> None: + staging = STATE_DIR / "finalize-output" + shutil.rmtree(staging, ignore_errors=True) + session = _session_from_state(state) + provider.finalize(session, staging) + recording = staging / "data" / "recording.mp4" + if recording.is_file(): + dest = DATA_DIR / "recording.mp4" + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(recording), dest) + state["recording_bytes"] = dest.stat().st_size + + +def _session_from_state(state: dict[str, Any]) -> BrowserSession: + metadata = state.get("metadata", {}) + inner = ( + metadata.get("metadata") if isinstance(metadata.get("metadata"), dict) else {} + ) + return BrowserSession( + provider="kernel", + mode="remote", + session_id=metadata.get("session_id"), + cdp_url=metadata.get("cdp_url", ""), + metadata={ + "replay_id": inner.get("replay_id") or metadata.get("replay_id"), + }, + recording_mode="provider-download", + ) + + +def cmd_finalize() -> int: + state = _load_state() + if state is None: + print("No Kernel browser was created; nothing to finalize", flush=True) + return 0 + if state.get("status") == "deleted": + print("Kernel browser already finalized and deleted", flush=True) + return 0 + + provider = _load_provider() + session_id = state.get("metadata", {}).get("session_id") + try: + _download_recording(provider, state) + _record(state, "replay_finalized", recording_bytes=state.get("recording_bytes")) + state["status"] = "finalized" + _write_metadata(state) + except BrowserRuntimeError as e: + # The replay is lost, but deletion must still proceed. + state["cleanup_error"] = f"replay finalization failed: {e}" + _record(state, "replay_finalization_failed", error=str(e)) + + try: + provider.cleanup(_session_from_state(state)) + state["cleanup_status"] = "deleted" + except BrowserRuntimeError as e: + state["cleanup_error"] = str(e) + _record(state, "finalize_failed", error=str(e)) + _save_state(state) + _write_metadata(state) + return 1 + + _verify_deletion(provider, state) + state["status"] = "deleted" + _record( + state, + "browser_deleted", + session_id=session_id, + cleanup_status=state.get("cleanup_status"), + deletion_verified=state.get("deletion_verified"), + ) + _save_state(state) + _write_metadata(state) + _write_lifecycle(state) + return 0 + + +def cmd_cleanup() -> int: + """Cleanup-only path (e.g. setup failed after the browser was created).""" + state = _load_state() + if state is None or state.get("status") == "deleted": + print("No Kernel browser to clean up", flush=True) + return 0 + + provider = _load_provider() + try: + provider.cleanup(_session_from_state(state)) + state["cleanup_status"] = "deleted" + except BrowserRuntimeError as e: + state["cleanup_error"] = str(e) + _record(state, "cleanup_failed", error=str(e)) + _write_metadata(state) + return 1 + + _verify_deletion(provider, state) + state["status"] = "deleted" + _record(state, "browser_deleted", cleanup_status=state.get("cleanup_status")) + _save_state(state) + _write_metadata(state) + _write_lifecycle(state) + return 0 + + +def _verify_deletion(provider: KernelRuntimeProvider, state: dict[str, Any]) -> None: + session_id = state.get("metadata", {}).get("session_id") + if not session_id: + state["deletion_verified"] = False + return + try: + state["deletion_verified"] = not provider.session_exists(session_id) + except BrowserRuntimeError as e: + state["deletion_verified"] = False + state["cleanup_error"] = f"deletion check failed: {e}" + + +def _write_lifecycle(state: dict[str, Any]) -> None: + LIFECYCLE_FILE.parent.mkdir(parents=True, exist_ok=True) + LIFECYCLE_FILE.write_text(json.dumps(_public_metadata(state), indent=2)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["start", "finalize", "cleanup"]) + args = parser.parse_args() + return {"start": cmd_start, "finalize": cmd_finalize, "cleanup": cmd_cleanup}[ + args.command + ]() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/clawbench/runtime/harbor/start-runtime.sh b/src/clawbench/runtime/harbor/start-runtime.sh index 2e0749d3..9232b87f 100644 --- a/src/clawbench/runtime/harbor/start-runtime.sh +++ b/src/clawbench/runtime/harbor/start-runtime.sh @@ -8,16 +8,32 @@ if [ -f /tmp/clawbench-run/runtime.pid ] && kill -0 "$(cat /tmp/clawbench-run/ru exit 0 fi -export DISPLAY="${DISPLAY:-:99}" -Xvfb "$DISPLAY" -screen 0 1920x1080x24 >/tmp/clawbench-run/xvfb.log 2>&1 & -echo "$!" > /tmp/clawbench-run/xvfb.pid -sleep 1 +# Remote browser runtimes (e.g. Kernel) hand us a provider CDP URL through a +# file instead of launching local Chromium. The runtime server proxies that +# WebSocket behind a credential-free local bridge on port 7878. +REMOTE_MODE=false +if [ -n "${CLAWBENCH_BROWSER_CDP_URL_FILE:-}" ] && [ -f "${CLAWBENCH_BROWSER_CDP_URL_FILE}" ]; then + REMOTE_MODE=true +fi cd /app/src/runtime-server uv run --no-sync uvicorn server:app --host 0.0.0.0 --port 7878 >/tmp/clawbench-run/runtime-server.log 2>&1 & echo "$!" > /tmp/clawbench-run/runtime-server.pid sleep 1 +if [ "$REMOTE_MODE" = true ]; then + # Provider replays replace local X11 recording. + export CLAWBENCH_RECORDING_MODE="${CLAWBENCH_RECORDING_MODE:-provider-download}" + echo "CDP bridge ready at http://127.0.0.1:7878" + echo "$$" > /tmp/clawbench-run/runtime.pid + exit 0 +fi + +export DISPLAY="${DISPLAY:-:99}" +Xvfb "$DISPLAY" -screen 0 1920x1080x24 >/tmp/clawbench-run/xvfb.log 2>&1 & +echo "$!" > /tmp/clawbench-run/xvfb.pid +sleep 1 + mkdir -p /tmp/chrome-profile/Default cat > /tmp/chrome-profile/Default/Preferences <<'PREFS' { diff --git a/src/clawbench/runtime/harbor/verify.py b/src/clawbench/runtime/harbor/verify.py index 3503f56d..2af1339c 100644 --- a/src/clawbench/runtime/harbor/verify.py +++ b/src/clawbench/runtime/harbor/verify.py @@ -5,10 +5,11 @@ import os import time import urllib.request +from concurrent.futures import ThreadPoolExecutor from pathlib import Path from typing import Any -JUDGE_SYSTEM = """You are a strict evaluator for a web-agent benchmark. +JUDGE_SYSTEM_STRICT = """You are a strict evaluator for a web-agent benchmark. A user gave a natural-language INSTRUCTION to an autonomous browser agent. The agent navigated a real website and finally produced a single HTTP request @@ -28,6 +29,49 @@ {"match": true|false, "reason": ""} """ +JUDGE_SYSTEM_LENIENT = """You are a lenient evaluator for a web-agent benchmark. + +A user gave a natural-language INSTRUCTION to an autonomous browser agent. +The agent navigated a real website and finally produced a single HTTP request +(URL + method + body) that we INTERCEPTED before it was sent to the server. + +You decide whether this intercepted request ALIGNS with the user's instruction. + +# Rubric (CRITICAL — read carefully) + +match = TRUE if there is NO EXPLICIT CONTRADICTION between the request and +the instruction. The default verdict is TRUE. + +match = FALSE only if the request EXPLICITLY contradicts a field the user +EXPLICITLY specified in the instruction. Examples of contradiction: +- Instruction says "red shirt", request body shows "color: blue" +- Instruction says "quantity 2", request body shows "qty: 5" +- Instruction says "deliver to home", request body shows "deliver to work" +- Instruction asks to enroll in CS50, request enrolls in CS61 + +# What is NOT a contradiction (default to match = TRUE) + +- Instruction did not specify a field, request omits or has any value for it + → MATCH (the user did not constrain it) +- Instruction said "3pm", request body has no time field at all + → MATCH (info absent, not contradicted) +- PDF resume left blank where instruction did not require content + → MATCH +- Cosmetic differences (timestamps, session IDs, affiliate codes, currency + symbols, formatting) → MATCH +- Ambiguous wording where multiple interpretations work → MATCH +- Agent picked a reasonable default for unspecified options → MATCH +- Color, size, time, quantity not mentioned in instruction → MATCH + +# Output + +Reply with ONLY a single-line JSON object, no markdown fences, no extra prose: +{"match": true|false, "reason": ""} + +Default is true. Only return false when you can name a SPECIFIC explicit +field from the instruction that the request EXPLICITLY contradicts. +""" + def write_reward( reward: float, payload: dict[str, Any], output_dir: Path = Path("/logs/verifier") @@ -35,8 +79,23 @@ def write_reward( out = output_dir out.mkdir(parents=True, exist_ok=True) result = {"reward": reward, **payload} + metrics = { + "reward": reward, + "intercepted": float(bool(payload.get("intercepted"))), + } + for name in ( + "reward_lenient", + "reward_strict", + "judge_match", + "judge_match_lenient", + "judge_match_strict", + ): + value = payload.get(name) + if isinstance(value, (bool, int, float)): + metrics[name] = float(value) + (out / "reward.txt").write_text(str(reward)) - (out / "reward.json").write_text(json.dumps(result, indent=2, ensure_ascii=False)) + (out / "reward.json").write_text(json.dumps(metrics, indent=2)) (out / "clawbench-result.json").write_text( json.dumps(result, indent=2, ensure_ascii=False) ) @@ -104,6 +163,8 @@ def call_judge( instruction: str, intercept: dict[str, Any], judge_context: dict[str, Any] | None, + system_prompt: str, + max_tokens: int, ) -> dict[str, Any]: api_type = model_cfg["api_type"] model = model_cfg["model"] @@ -116,10 +177,10 @@ def call_judge( { "model": model, "messages": [ - {"role": "system", "content": JUDGE_SYSTEM}, + {"role": "system", "content": system_prompt}, {"role": "user", "content": user}, ], - "max_tokens": 4096, + "max_tokens": max_tokens, "temperature": 0, }, ) @@ -130,9 +191,9 @@ def call_judge( {"Authorization": f"Bearer {model_cfg['api_key']}"}, { "model": model, - "instructions": JUDGE_SYSTEM, + "instructions": system_prompt, "input": user, - "max_output_tokens": 4096, + "max_output_tokens": max_tokens, }, ) raw = resp.get("output_text") or "" @@ -152,8 +213,8 @@ def call_judge( }, { "model": model, - "max_tokens": 4096, - "system": JUDGE_SYSTEM, + "max_tokens": max_tokens, + "system": system_prompt, "messages": [{"role": "user", "content": user}], }, ) @@ -169,6 +230,37 @@ def call_judge( return {"match": match, "reason": reason, "raw": raw, "error": None} +def call_judge_with_retries( + model_cfg: dict[str, str], + instruction: str, + intercept: dict[str, Any], + judge_context: dict[str, Any] | None, + system_prompt: str, + max_tokens: int, +) -> dict[str, Any]: + last_error = "" + for attempt in range(3): + try: + return call_judge( + model_cfg, + instruction, + intercept, + judge_context, + system_prompt, + max_tokens, + ) + except Exception as exc: + last_error = str(exc) + if attempt < 2: + time.sleep(2**attempt) + return { + "match": None, + "reason": f"judge_call_failed: {last_error}", + "raw": None, + "error": last_error, + } + + def main() -> int: task_path = Path("/tests/task.json") intercept_path = Path("/data/interception.json") @@ -185,8 +277,12 @@ def main() -> int: write_reward( 0.0, { + "reward_lenient": 0.0, + "reward_strict": 0.0, "intercepted": False, "judge_match": None, + "judge_match_lenient": None, + "judge_match_strict": None, "reason": "missing /data/interception.json", "task_id": task_id, }, @@ -198,8 +294,12 @@ def main() -> int: write_reward( 0.0, { + "reward_lenient": 0.0, + "reward_strict": 0.0, "intercepted": False, "judge_match": None, + "judge_match_lenient": None, + "judge_match_strict": None, "reason": intercept.get("stop_description") or intercept.get("stop_reason") or "not intercepted", @@ -218,54 +318,69 @@ def main() -> int: write_reward( 0.0, { + "reward_lenient": 0.0, + "reward_strict": 0.0, "intercepted": True, "judge_match": None, + "judge_match_lenient": None, + "judge_match_strict": None, "reason": "missing judge configuration", "task_id": task_id, }, ) return 0 - judge_result: dict[str, Any] | None = None - last_error = "" - for attempt in range(3): - try: - judge_result = call_judge( - cfg, - str(task.get("instruction") or ""), - intercept, - task.get("judge_context") - if isinstance(task.get("judge_context"), dict) - else None, - ) - break - except Exception as exc: - last_error = str(exc) - if attempt < 2: - time.sleep(2**attempt) - - if judge_result is None: - write_reward( - 0.0, - { - "intercepted": True, - "judge_match": None, - "reason": f"judge_call_failed: {last_error}", - "task_id": task_id, - }, + instruction = str(task.get("instruction") or "") + judge_context = ( + task.get("judge_context") + if isinstance(task.get("judge_context"), dict) + else None + ) + with ThreadPoolExecutor(max_workers=2) as executor: + lenient_future = executor.submit( + call_judge_with_retries, + cfg, + instruction, + intercept, + None, + JUDGE_SYSTEM_LENIENT, + 800, ) - return 0 + strict_future = executor.submit( + call_judge_with_retries, + cfg, + instruction, + intercept, + judge_context, + JUDGE_SYSTEM_STRICT, + 4096, + ) + lenient_result = lenient_future.result() + strict_result = strict_future.result() - match = judge_result.get("match") - reward = 1.0 if match is True else 0.0 + match_lenient = lenient_result.get("match") + match_strict = strict_result.get("match") + reward_lenient = 1.0 if match_lenient is True else 0.0 + reward_strict = 1.0 if match_strict is True else 0.0 write_reward( - reward, + reward_lenient, { + "reward_lenient": reward_lenient, + "reward_strict": reward_strict, "intercepted": True, - "judge_match": match, - "reason": judge_result.get("reason") or judge_result.get("error") or "", - "task_id": task_id, + "judge_match": match_lenient, + "judge_match_lenient": match_lenient, + "judge_match_strict": match_strict, + "reason": { + "lenient": lenient_result.get("reason") + or lenient_result.get("error") + or "", + "strict": strict_result.get("reason") + or strict_result.get("error") + or "", + }, "judge_model": cfg["model"], + "task_id": task_id, }, ) return 0 diff --git a/src/clawbench/runtime/harbor/wrap-harbor-agent.sh b/src/clawbench/runtime/harbor/wrap-harbor-agent.sh new file mode 100755 index 00000000..50cb7509 --- /dev/null +++ b/src/clawbench/runtime/harbor/wrap-harbor-agent.sh @@ -0,0 +1,70 @@ +#!/bin/bash +set -euo pipefail + +wrap_agent() { + local executable=$1 + local path real + + path=$(command -v "$executable" 2>/dev/null || true) + if [[ -z "$path" && -x "$HOME/.local/bin/$executable" ]]; then + path="$HOME/.local/bin/$executable" + fi + [[ -n "$path" && -x "$path" ]] || return 0 + + real="${path}.clawbench-real" + if [[ ! -e "$real" ]]; then + mv "$path" "$real" + fi + + cat >"$path" <<'WRAPPER' +#!/bin/bash +set +e + +real="${BASH_SOURCE[0]}.clawbench-real" +stop_file=${CLAWBENCH_STOP_FILE:-/data/.stop-requested} +stop_result=${CLAWBENCH_STOP_RESULT:-/data/agent-stop.json} +rm -f "$stop_file" "$stop_result" + +"$real" "$@" <&0 & +agent_pid=$! +( + while kill -0 "$agent_pid" 2>/dev/null; do + if [[ -f "$stop_file" ]]; then + detected_at=$(date +%s.%N) + printf '{"stop_detected_at":%s,"signal":"INT"}\n' "$detected_at" >"$stop_result" + kill -INT "$agent_pid" 2>/dev/null || true + for _ in $(seq 1 20); do + kill -0 "$agent_pid" 2>/dev/null || break + sleep 0.1 + done + if kill -0 "$agent_pid" 2>/dev/null; then + kill -TERM "$agent_pid" 2>/dev/null || true + fi + exit 0 + fi + sleep 0.1 + done +) & +watcher_pid=$! + +wait "$agent_pid" +status=$? +kill "$watcher_pid" 2>/dev/null || true +wait "$watcher_pid" 2>/dev/null || true + +if [[ -f "$stop_file" ]]; then + exit 0 +fi +exit "$status" +WRAPPER + chmod 0755 "$path" +} + +if (( $# > 0 )); then + for executable in "$@"; do + wrap_agent "$executable" + done +else + wrap_agent claude + wrap_agent codex +fi diff --git a/src/clawbench/runtime/runtime-server/server.py b/src/clawbench/runtime/runtime-server/server.py index 3de37cbe..5b9c614d 100644 --- a/src/clawbench/runtime/runtime-server/server.py +++ b/src/clawbench/runtime/runtime-server/server.py @@ -493,6 +493,7 @@ def activate_session_target(session_id, reason): if not INTERCEPTION_FILE.exists(): result = { "intercepted": True, + "intercepted_at": time.time(), "request": request_obj, "schema": eval_schema, } diff --git a/test-cases/v2/v2-1134-chapter-finder-redcross/task.json b/test-cases/v2/v2-1134-chapter-finder-redcross/task.json index 57acd1b8..aaf54986 100644 --- a/test-cases/v2/v2-1134-chapter-finder-redcross/task.json +++ b/test-cases/v2/v2-1134-chapter-finder-redcross/task.json @@ -17,8 +17,11 @@ }, "instruction": "Find the nearest Red Cross chapter to zip code 90210 using the local chapter finder on redcross.org", "eval_schema": { - "url_pattern": "redcross\\.org\\/find-your-local-chapter", - "method": "GET" + "url_pattern": "redcross\\.org/api/lookup/v1/region-mappings/90210(?:\\?|$)", + "method": "GET", + "params": { + "type": "RCO" + } }, "time_limit": 30, "extra_info": [] diff --git a/tests/test_browser_runtime.py b/tests/test_browser_runtime.py index 11dccdd0..85e6568c 100644 --- a/tests/test_browser_runtime.py +++ b/tests/test_browser_runtime.py @@ -398,7 +398,7 @@ def fake_urlopen( monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) provider = KernelRuntimeProvider( api_key="kernel-secret", - options={"stealth": True, "region": "us-east"}, + options={"region": "us-east"}, replay_poll_interval_s=0, replay_poll_timeout_s=1, ) @@ -413,7 +413,6 @@ def fake_urlopen( "region": "us-east", "headless": False, "timeout_seconds": 1920, - "viewport": {"width": 1920, "height": 1080, "refresh_rate": 25}, } assert session.provider == "kernel" assert session.recording_mode == "provider-download" diff --git a/tests/test_harbor_adapter.py b/tests/test_harbor_adapter.py index af67bee5..c7afe7b8 100644 --- a/tests/test_harbor_adapter.py +++ b/tests/test_harbor_adapter.py @@ -14,6 +14,7 @@ unique_output_name, write_harbor_task, ) +from clawbench.runtime.harbor import verify as harbor_verify from clawbench.runtime.harbor.verify import write_reward SRC_ROOT = Path(__file__).resolve().parents[1] / "src" @@ -108,11 +109,12 @@ def test_write_harbor_task_emits_expected_tree_and_extra_info(tmp_path: Path) -> assert ( config["task"]["name"] == "clawbench/v2-047-daily-life-personal-care-taskrabbit" ) - assert config["environment"]["workdir"] == "/app" + assert config["environment"]["workdir"] == "/" assert config["environment"]["network_mode"] == "public" assert config["environment"]["env"]["BROWSER_CDP_URL"] == "http://127.0.0.1:9223" assert config["environment"]["env"]["PLAYWRIGHT_CDP_URL"] == "http://127.0.0.1:9223" assert config["steps"][0]["agent"]["timeout_sec"] == 1800.0 + assert config["steps"][0]["verifier"]["timeout_sec"] == 300.0 assert ( "http://127.0.0.1:9223" in (out / "steps" / "run" / "instruction.md").read_text() @@ -172,7 +174,7 @@ def test_harbor_adapter_help_without_container_runtime(tmp_path: Path) -> None: assert "usage" in (result.stdout + result.stderr).lower() -def test_harbor_verifier_reward_json_contains_metadata(tmp_path: Path) -> None: +def test_harbor_verifier_separates_metrics_from_metadata(tmp_path: Path) -> None: write_reward( 0.0, { @@ -187,11 +189,99 @@ def test_harbor_verifier_reward_json_contains_metadata(tmp_path: Path) -> None: assert (tmp_path / "reward.txt").read_text() == "0.0" reward = json.loads((tmp_path / "reward.json").read_text()) detailed = json.loads((tmp_path / "clawbench-result.json").read_text()) - assert reward == detailed assert reward == { + "reward": 0.0, + "intercepted": 1.0, + "judge_match": 0.0, + } + assert detailed == { "reward": 0.0, "intercepted": True, "judge_match": False, "reason": "wrong payload", "task_id": 47, } + + +def test_harbor_judge_supports_lenient_and_strict_prompts(monkeypatch) -> None: + requests = [] + + def fake_post_json(url, headers, payload, timeout=60): + requests.append(payload) + return {"choices": [{"message": {"content": '{"match": true}'}}]} + + monkeypatch.setattr(harbor_verify, "post_json", fake_post_json) + config = { + "api_type": "openai-completions", + "model": "judge", + "base_url": "https://judge.example/v1", + "api_key": "key", + } + intercept = {"request": {"url": "https://example.com/submit", "method": "POST"}} + + harbor_verify.call_judge( + config, + "submit the form", + intercept, + None, + harbor_verify.JUDGE_SYSTEM_LENIENT, + 800, + ) + harbor_verify.call_judge( + config, + "submit the form", + intercept, + {"rubric": "all fields are required"}, + harbor_verify.JUDGE_SYSTEM_STRICT, + 4096, + ) + + assert requests[0]["messages"][0]["content"] == harbor_verify.JUDGE_SYSTEM_LENIENT + assert requests[0]["max_tokens"] == 800 + assert requests[1]["messages"][0]["content"] == harbor_verify.JUDGE_SYSTEM_STRICT + assert requests[1]["max_tokens"] == 4096 + assert "all fields are required" in requests[1]["messages"][1]["content"] + + +def test_harbor_verifier_emits_both_reward_rubrics(tmp_path: Path) -> None: + write_reward( + 1.0, + { + "reward_lenient": 1.0, + "reward_strict": 0.0, + "intercepted": True, + "judge_match": True, + "judge_match_lenient": True, + "judge_match_strict": False, + "reason": {"lenient": "no contradiction", "strict": "missing time"}, + "task_id": 47, + }, + output_dir=tmp_path, + ) + + reward = json.loads((tmp_path / "reward.json").read_text()) + assert reward == { + "reward": 1.0, + "intercepted": 1.0, + "reward_lenient": 1.0, + "reward_strict": 0.0, + "judge_match": 1.0, + "judge_match_lenient": 1.0, + "judge_match_strict": 0.0, + } + + +def test_harbor_verifier_omits_unknown_judge_match_metric(tmp_path: Path) -> None: + write_reward( + 0.0, + { + "intercepted": False, + "judge_match": None, + "reason": "not intercepted", + "task_id": 47, + }, + output_dir=tmp_path, + ) + + reward = json.loads((tmp_path / "reward.json").read_text()) + assert reward == {"reward": 0.0, "intercepted": 0.0} diff --git a/tests/test_harbor_kernel_control.py b/tests/test_harbor_kernel_control.py new file mode 100644 index 00000000..d4bc621b --- /dev/null +++ b/tests/test_harbor_kernel_control.py @@ -0,0 +1,497 @@ +"""Regression tests for the Harbor ClawBench Kernel control arm.""" + +from __future__ import annotations + +import importlib.util +import json +import re +import subprocess +import sys +import time +import tomllib +from pathlib import Path + +import pytest + +from clawbench.eval.harbor_adapter import ( + PLAYWRIGHT_MCP_PACKAGE, + PLAYWRIGHT_MCP_VERSION, + main as adapt_main, + write_harbor_task, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] +KERNEL_BROWSER_SCRIPT = ( + REPO_ROOT / "src" / "clawbench" / "runtime" / "harbor" / "kernel-browser.py" +) +HARBOR_AGENT_WRAPPER = ( + REPO_ROOT / "src" / "clawbench" / "runtime" / "harbor" / "wrap-harbor-agent.sh" +) + + +def _task() -> dict: + return { + "metadata": { + "task_id": 1134, + "description": "Find nearest Red Cross chapter", + }, + "instruction": "Find the nearest Red Cross chapter to zip code 90210.", + "eval_schema": {"url_pattern": "redcross\\.org/chapter", "method": "GET"}, + "time_limit": 30, + "extra_info": [], + } + + +@pytest.fixture() +def adapted_kernel_task(tmp_path: Path) -> Path: + case = tmp_path / "case" + case.mkdir() + (case / "task.json").write_text(json.dumps(_task())) + return write_harbor_task( + task_dir=case, + task=_task(), + output_root=tmp_path / "out", + output_name="v2-1134-chapter-finder-redcross", + org="clawbench", + dataset_name="v2", + browser_runtime="kernel", + ) + + +def test_redcross_task_intercepts_zip_lookup_not_chapter_finder_page() -> None: + task_path = ( + REPO_ROOT + / "test-cases" + / "v2" + / "v2-1134-chapter-finder-redcross" + / "task.json" + ) + schema = json.loads(task_path.read_text())["eval_schema"] + + assert not re.search( + schema["url_pattern"], + "https://www.redcross.org/find-your-local-chapter.html", + ) + assert re.search( + schema["url_pattern"], + "https://www.redcross.org/api/lookup/v1/region-mappings/90210?type=RCO", + ) + assert schema["params"] == {"type": "RCO"} + + +def test_kernel_runtime_selection_writes_bridge_env_and_pinned_mcp( + adapted_kernel_task: Path, +) -> None: + config = tomllib.loads((adapted_kernel_task / "task.toml").read_text()) + + assert config["metadata"]["browser_runtime"] == "kernel" + env = config["environment"]["env"] + assert env["CLAWBENCH_HARBOR_BROWSER_RUNTIME"] == "kernel" + assert env["PLAYWRIGHT_CDP_URL"] == "http://127.0.0.1:7878" + assert env["CLAWBENCH_CDP_URL"] == "http://127.0.0.1:7878" + assert env["CLAWBENCH_RECORDING_MODE"] == "provider-download" + assert env["KERNEL_API_KEY"] == "${KERNEL_API_KEY}" + + servers = config["environment"]["mcp_servers"] + assert len(servers) == 1 + server = servers[0] + assert server["name"] == "playwright" + assert server["transport"] == "stdio" + assert server["command"] == "npx" + assert server["args"][0] == "-y" + # Pinned package so the control arm is reproducible. + assert server["args"][1] == f"{PLAYWRIGHT_MCP_PACKAGE}@{PLAYWRIGHT_MCP_VERSION}" + assert "--cdp-endpoint" in server["args"] + assert "http://127.0.0.1:7878" in server["args"] + + healthcheck = config["steps"][0]["healthcheck"]["command"] + assert "7878/json/version" in healthcheck + + +def test_kernel_setup_and_test_scripts_wire_lifecycle( + adapted_kernel_task: Path, +) -> None: + workdir = adapted_kernel_task / "steps" / "run" / "workdir" + tests = adapted_kernel_task / "steps" / "run" / "tests" + + setup = (workdir / "setup.sh").read_text() + assert "kernel-browser.py start" in setup + assert "export CLAWBENCH_BROWSER_CDP_URL_FILE=" in setup + assert "trap cleanup_browser EXIT" in setup + assert "kernel-browser.py cleanup" in setup + assert "wrap-harbor-agent.sh" in setup + assert "127.0.0.1:7878/json/version" in setup + + test = (tests / "test.sh").read_text() + assert "kernel-browser.py finalize" in test + + env_dir = adapted_kernel_task / "environment" + assert (env_dir / "harbor" / "kernel-browser.py").is_file() + assert (env_dir / "harbor" / "browser_runtime_providers.py").is_file() + + +@pytest.mark.skipif( + sys.platform != "linux", + reason="wrap-harbor-agent.sh runs inside Harbor's Linux containers; " + "bash signal handling differs on mac and windows runners", +) +def test_stop_wrapper_interrupts_agent_and_exits_cleanly(tmp_path: Path) -> None: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + fake_agent = bin_dir / "claude" + started = tmp_path / "started" + stop_file = tmp_path / "stop-requested" + stop_result = tmp_path / "agent-stop.json" + fake_agent.write_text( + "#!/bin/bash\n" + "trap 'exit 130' INT\n" + 'touch "$FAKE_AGENT_STARTED"\n' + "while true; do sleep 0.1; done\n" + ) + fake_agent.chmod(0o755) + env = { + "PATH": f"{bin_dir}:/usr/bin:/bin", + "HOME": str(tmp_path), + "FAKE_AGENT_STARTED": str(started), + "CLAWBENCH_STOP_FILE": str(stop_file), + "CLAWBENCH_STOP_RESULT": str(stop_result), + } + + subprocess.run([HARBOR_AGENT_WRAPPER, "claude"], env=env, check=True) + process = subprocess.Popen([fake_agent], env=env) + deadline = time.monotonic() + 2 + while not started.exists() and time.monotonic() < deadline: + time.sleep(0.01) + assert started.exists() + + stop_file.touch() + assert process.wait(timeout=3) == 0 + assert json.loads(stop_result.read_text())["signal"] == "INT" + + +def test_local_runtime_default_has_no_kernel_hooks(tmp_path: Path) -> None: + case = tmp_path / "case" + case.mkdir() + (case / "task.json").write_text(json.dumps(_task())) + out = write_harbor_task( + task_dir=case, + task=_task(), + output_root=tmp_path / "out", + output_name="local", + org="clawbench", + dataset_name="v2", + ) + + config = tomllib.loads((out / "task.toml").read_text()) + assert config["metadata"]["browser_runtime"] == "local" + assert "mcp_servers" not in config["environment"] + assert config["environment"]["env"]["PLAYWRIGHT_CDP_URL"] == "http://127.0.0.1:9223" + assert config["environment"]["env"].get("CLAWBENCH_HARBOR_BROWSER_RUNTIME") is None + + setup = (out / "steps" / "run" / "workdir" / "setup.sh").read_text() + assert "kernel-browser" not in setup + assert (out / "environment" / "harbor" / "browser_runtime_providers.py").is_file() + + +def test_instruction_ports_native_restrictions(adapted_kernel_task: Path) -> None: + instruction = (adapted_kernel_task / "steps" / "run" / "instruction.md").read_text() + # Native ClawBench prompt text (source of truth) survives verbatim. + assert "entirely through the browser" in instruction + assert "Do NOT use command-line tools, scripts, or direct API/SMTP calls" in ( + instruction + ) + # Ported harness restrictions. + assert "Time limit: 30 minutes" in instruction + assert "Playwright MCP browser tools" in instruction + assert "Do NOT make direct HTTP/network requests" in instruction + assert "Submit through the browser" in instruction + assert "Stop after submission" in instruction + # Credential-free bridge endpoint, never a provider URL. + assert "http://127.0.0.1:7878" in instruction + assert "ws://" not in instruction + + +def test_adapter_cli_rejects_invalid_runtime_options_json(tmp_path: Path) -> None: + with pytest.raises(SystemExit) as excinfo: + adapt_main( + [ + "--output-dir", + str(tmp_path / "out"), + "--limit", + "1", + "--overwrite", + "--browser-runtime", + "kernel", + "--browser-runtime-options", + "not-json", + ] + ) + assert excinfo.value.code == 2 + + +def test_adapter_cli_bakes_runtime_options_into_task_toml( + tmp_path: Path, +) -> None: + adapt_main( + [ + "--output-dir", + str(tmp_path / "out"), + "--limit", + "1", + "--overwrite", + "--browser-runtime", + "kernel", + "--browser-runtime-options", + '{"stealth": true}', + ] + ) + task_toml = next((tmp_path / "out").glob("*/task.toml")) + config = tomllib.loads(task_toml.read_text()) + assert config["environment"]["env"]["CLAWBENCH_BROWSER_RUNTIME_OPTIONS"] == ( + '{"stealth": true}' + ) + + +# --------------------------------------------------------------------------- +# kernel-browser.py lifecycle script +# --------------------------------------------------------------------------- + + +def _load_kernel_browser_module(monkeypatch: pytest.MonkeyPatch, home: Path): + spec = importlib.util.spec_from_file_location( + "kernel_browser_under_test", KERNEL_BROWSER_SCRIPT + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "STATE_DIR", home / "clawbench-run") + monkeypatch.setattr( + module, "STATE_FILE", home / "clawbench-run" / "kernel-browser-state.json" + ) + monkeypatch.setattr( + module, "CDP_URL_FILE", home / "clawbench-run" / "kernel-cdp-url" + ) + monkeypatch.setattr( + module, "METADATA_FILE", home / "my-info" / "kernel_browser.json" + ) + monkeypatch.setattr( + module, "LIFECYCLE_FILE", home / "data" / "kernel-browser-lifecycle.json" + ) + monkeypatch.setattr(module, "TASK_FILE", home / "task.json") + monkeypatch.setattr(module, "DATA_DIR", home / "data") + sys.modules[spec.name] = module + return module + + +class FakeSession: + def __init__(self) -> None: + from clawbench.runner.run_support.browser_runtime.providers import ( + BrowserSession, + ) + + self._session = BrowserSession( + provider="kernel", + mode="remote", + session_id="sess-123", + cdp_url="wss://kernel.example/browser/sess-123/cdp?token=secret-token", + viewer_url="https://kernel.example/live/sess-123", + viewer_url_sensitive=True, + metadata={ + "replay_id": "replay-9", + "region": "us-east", + "stealth": False, + }, + recording_mode="provider-download", + ) + + def __getattr__(self, name: str): + return getattr(self._session, name) + + +class FakeProvider: + name = "kernel" + + def __init__(self, fail_cleanup: bool = False) -> None: + self.fail_cleanup = fail_cleanup + self.finalize_calls = 0 + self.cleanup_calls = 0 + self.exists_calls: list[str] = [] + self.deleted = False + + def start(self, task: dict, time_limit_s: int) -> FakeSession: + assert time_limit_s == 1800 + return FakeSession() + + def finalize(self, session: FakeSession, output_dir: Path) -> None: + self.finalize_calls += 1 + recording = Path(output_dir) / "data" / "recording.mp4" + recording.parent.mkdir(parents=True, exist_ok=True) + recording.write_bytes(b"mp4-bytes") + session.metadata["recording_bytes"] = len(b"mp4-bytes") + + def cleanup(self, session: FakeSession) -> None: + self.cleanup_calls += 1 + if self.fail_cleanup: + from clawbench.runner.run_support.browser_runtime.providers import ( + BrowserRuntimeError, + ) + + raise BrowserRuntimeError("delete failed") + self.deleted = True + + def session_exists(self, session_id: str) -> bool: + self.exists_calls.append(session_id) + return not self.deleted + + +@pytest.fixture() +def kernel_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + (tmp_path / "task.json").write_text(json.dumps(_task())) + for var in ( + "KERNEL_API_KEY", + "KERNEL_BASE_URL", + "CLAWBENCH_BROWSER_RUNTIME_OPTIONS", + ): + monkeypatch.delenv(var, raising=False) + return tmp_path + + +def test_start_writes_state_and_credential_free_metadata( + kernel_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("KERNEL_API_KEY", "k-test-key") + module = _load_kernel_browser_module(monkeypatch, kernel_env) + provider = FakeProvider() + monkeypatch.setattr(module, "_load_provider", lambda: provider) + + assert module.cmd_start() == 0 + + state = json.loads(module.STATE_FILE.read_text()) + assert state["status"] == "created" + assert state["cdp_url"].startswith("wss://kernel.example") + if sys.platform != "win32": + assert oct(module.STATE_FILE.stat().st_mode & 0o777) == "0o600" + assert oct(module.CDP_URL_FILE.stat().st_mode & 0o777) == "0o600" + + metadata = json.loads(module.METADATA_FILE.read_text()) + assert metadata["session_id"] == "sess-123" + assert metadata["replay_id"] == "replay-9" + assert metadata["stealth"] is False + assert metadata["cdp_bridge_url"] == "http://127.0.0.1:7878" + blob = module.METADATA_FILE.read_text() + assert "k-test-key" not in blob + assert "secret-token" not in blob + + +def test_finalize_downloads_recording_deletes_browser_and_is_idempotent( + kernel_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load_kernel_browser_module(monkeypatch, kernel_env) + provider = FakeProvider() + monkeypatch.setattr(module, "_load_provider", lambda: provider) + assert module.cmd_start() == 0 + + assert module.cmd_finalize() == 0 + + recording = kernel_env / "data" / "recording.mp4" + assert recording.read_bytes() == b"mp4-bytes" + lifecycle = json.loads(module.LIFECYCLE_FILE.read_text()) + assert lifecycle["status"] == "deleted" + assert lifecycle["deletion_verified"] is True + assert provider.cleanup_calls == 1 + assert provider.exists_calls == ["sess-123"] + + # Idempotent rerun makes no further API calls. + assert module.cmd_finalize() == 0 + assert provider.cleanup_calls == 1 + + +def test_finalize_reports_failure_when_cleanup_fails( + kernel_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load_kernel_browser_module(monkeypatch, kernel_env) + provider = FakeProvider(fail_cleanup=True) + monkeypatch.setattr(module, "_load_provider", lambda: provider) + assert module.cmd_start() == 0 + + assert module.cmd_finalize() == 1 + state = json.loads(module.STATE_FILE.read_text()) + assert state["cleanup_error"] == "delete failed" + + +def test_cleanup_recovers_from_agent_visible_metadata_alone( + kernel_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load_kernel_browser_module(monkeypatch, kernel_env) + provider = FakeProvider() + monkeypatch.setattr(module, "_load_provider", lambda: provider) + assert module.cmd_start() == 0 + module.STATE_FILE.unlink() + + assert module.cmd_cleanup() == 0 + assert provider.cleanup_calls == 1 + lifecycle = json.loads(module.LIFECYCLE_FILE.read_text()) + assert lifecycle["status"] == "deleted" + assert lifecycle["deletion_verified"] is True + + +def test_commands_are_noops_without_a_created_browser( + kernel_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load_kernel_browser_module(monkeypatch, kernel_env) + provider = FakeProvider() + monkeypatch.setattr(module, "_load_provider", lambda: provider) + + assert module.cmd_finalize() == 0 + assert module.cmd_cleanup() == 0 + assert provider.cleanup_calls == 0 + + +def test_kernel_session_exists_reports_404_as_missing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from clawbench.runner.run_support.browser_runtime.providers import ( + KernelRuntimeProvider, + _KernelApiError, + ) + + provider = KernelRuntimeProvider(api_key="k", options={}) + + def fake_request(method: str, path: str, payload=None) -> bytes: + raise _KernelApiError("not found", status=404) + + monkeypatch.setattr(provider, "_request", fake_request) + assert provider.session_exists("gone") is False + + def ok_request(method: str, path: str, payload=None) -> bytes: + if method == "GET": + return b"{}" + raise AssertionError(method) + + monkeypatch.setattr(provider, "_request", ok_request) + assert provider.session_exists("alive") is True + + +def test_finalize_still_deletes_browser_when_replay_download_fails( + kernel_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from clawbench.runner.run_support.browser_runtime.providers import ( + BrowserRuntimeError, + ) + + module = _load_kernel_browser_module(monkeypatch, kernel_env) + provider = FakeProvider() + monkeypatch.setattr(module, "_load_provider", lambda: provider) + + def failing_finalize(session, output_dir): # noqa: ANN001 + raise BrowserRuntimeError("replay stuck processing") + + monkeypatch.setattr(provider, "finalize", failing_finalize) + assert module.cmd_start() == 0 + + assert module.cmd_finalize() == 0 + lifecycle = json.loads(module.LIFECYCLE_FILE.read_text()) + assert lifecycle["status"] == "deleted" + assert lifecycle["deletion_verified"] is True + assert "replay finalization failed" in lifecycle["cleanup_error"] + assert provider.cleanup_calls == 1