diff --git a/.github/workflows/_run.yml b/.github/workflows/_run.yml index 106f2ea..0b09d50 100644 --- a/.github/workflows/_run.yml +++ b/.github/workflows/_run.yml @@ -71,6 +71,8 @@ jobs: STEEL_API_KEY: ${{ secrets.STEEL_API_KEY }} HYPERBROWSER_API_KEY: ${{ secrets.HYPERBROWSER_API_KEY }} ANCHOR_API_KEY: ${{ secrets.ANCHOR_API_KEY }} + EV_APP_UUID: ${{ secrets.EV_APP_UUID }} + EV_API_KEY: ${{ secrets.EV_API_KEY }} BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} BRAINTRUST_PROJECT: ${{ vars.BRAINTRUST_PROJECT }} steps: diff --git a/README.md b/README.md index 10bb302..f8bd103 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,7 @@ A run of two arms on two tasks (Red Cross and Indeed) produced this summary: | `PURELY_MAIL_API_KEY`, `PURELY_MAIL_DOMAIN` | ClawBench creates a disposable [PurelyMail](https://purelymail.com) mailbox on your domain for every task, so agents can sign up and receive confirmation emails. | | `CLAWBENCH_JUDGE_BASE_URL`, `CLAWBENCH_JUDGE_API_KEY`, `CLAWBENCH_JUDGE_MODEL`, `CLAWBENCH_JUDGE_API_TYPE` | ClawBench's LLM judge, which checks the intercepted request against the task. Any OpenAI-compatible endpoint works with `CLAWBENCH_JUDGE_API_TYPE=openai-completions`; we use `deepseek/deepseek-v4-pro` through OpenRouter. See [ClawBench](https://github.com/TIGER-AI-Lab/ClawBench) for details. | | `BROWSERBASE_API_KEY`, `BROWSERBASE_PROJECT_ID`, `STEEL_API_KEY`, `HYPERBROWSER_API_KEY`, `ANCHOR_API_KEY` | Arms on those providers' browsers (`environment = "remote-browser"`); checked before a run that includes them | +| `EV_APP_UUID`, `EV_API_KEY` | `codex-pw-kernel-evervault`, which mints an Evervault Browser Tokens session per trial for its proxy. The key needs `browserToken:create`. | | `BRAINTRUST_API_KEY`, `BRAINTRUST_PROJECT` | Optional: publishing results | ## Running in GitHub Actions @@ -147,6 +148,7 @@ The `retail-cart` tasks on each browser provider's browser. Every arm uses the s | Arm | Browser | |---|---| | `codex-pw-kernel` | Kernel, `stealth: true` | +| `codex-pw-kernel-evervault` | Kernel, `stealth: true`, with the whole session sent through Evervault's browser proxy: a new Evervault session and Kernel custom proxy with Evervault's CA per trial | | `codex-pw-browserbase` | Browserbase, Advanced Stealth (Scale plan), CAPTCHA solving, its residential proxies | | `codex-pw-steel` | Steel, residential proxy and CAPTCHA solving | | `codex-pw-hyperbrowser` | Hyperbrowser, stealth, proxy and CAPTCHA solving | @@ -160,6 +162,21 @@ uv run bench run retail-cart-providers --attempts 10 --concurrency 4 All arms' task containers run on one machine, so keep `--concurrency` low on a GitHub runner. +### `retail-cart-geo` + +The `retail-cart` tasks reached from each site's own market, for sites outside the US. Tasks need a `country`, so pass a task file of the same shape as `suites/retail-cart/tasks.toml` with `--suite-arg tasks=`. + +| Arm | Browser | +|---|---| +| `codex-pw-kernel-geo` | Kernel, `stealth: true`, with a new residential proxy in the task's country per trial (the `geo` configurator) | +| `codex-pw-kernel-evervault-eu` | Kernel, `stealth: true`, with the whole session sent through Evervault's EU (Ireland) relay | +| `codex-pw-kernel-evervault-residential` | Same, with Evervault's residential egress in the task's country. Needs residential egress enabled on the Evervault app. | +| `codex-pw-kernel-registry` | Kernel, with the browser settings, proxy and guidance Kernel's config registry recommends for the task's start URL, limited to the task's country (the `registry` configurator). Sites without a recommendation get the `codex-pw-kernel-geo` settings. | + +```bash +uv run bench run retail-cart-geo --suite-arg tasks=path/to/tasks.toml --attempts 5 +``` + ## Extending ### Add an arm diff --git a/arms/codex-pw-kernel-evervault-eu.toml b/arms/codex-pw-kernel-evervault-eu.toml new file mode 100644 index 0000000..5c102c8 --- /dev/null +++ b/arms/codex-pw-kernel-evervault-eu.toml @@ -0,0 +1,23 @@ +# Same as codex-pw-kernel-evervault, but through Evervault's EU (Ireland) +# relay, for sites that serve European markets. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "kernel-evervault" + +[browser.config.params] +stealth = true +timeout_seconds = 1800 + +# See codex-pw-kernel-evervault for the TTL. +[browser.config.params.evervault] +ttl = 900 +hosts = ["checkout-test.adyen.com"] +relay_host = "eu-ie.browser.relay.evervault.app" diff --git a/arms/codex-pw-kernel-evervault-residential.toml b/arms/codex-pw-kernel-evervault-residential.toml new file mode 100644 index 0000000..ab7d938 --- /dev/null +++ b/arms/codex-pw-kernel-evervault-residential.toml @@ -0,0 +1,29 @@ +# Same as codex-pw-kernel-evervault-eu, but Evervault sends the session out +# through its residential egress in the country of the task's site (the +# `geo` configurator), as the purchase pipelines it is compared with do. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser] +configurator = "geo" +configurator_args = { proxy = "residential" } + +[browser.config] +provider = "kernel-evervault" + +[browser.config.params] +stealth = true +timeout_seconds = 1800 + +# See codex-pw-kernel-evervault for the TTL. Residential egress has to be +# enabled for the Evervault app. +[browser.config.params.evervault] +ttl = 900 +hosts = ["checkout-test.adyen.com"] +relay_host = "eu-ie.browser.relay.evervault.app" diff --git a/arms/codex-pw-kernel-evervault.toml b/arms/codex-pw-kernel-evervault.toml new file mode 100644 index 0000000..612c60c --- /dev/null +++ b/arms/codex-pw-kernel-evervault.toml @@ -0,0 +1,26 @@ +# Same as codex-pw-kernel, but the whole browser session goes through +# Evervault's browser proxy, so sites see Evervault's egress instead of +# Kernel's stealth proxy pool. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser.config] +provider = "kernel-evervault" + +[browser.config.params] +stealth = true +timeout_seconds = 1800 + +# The proxy closes its tunnels when the mint's TTL ends. Sandbox apps cap the +# TTL at 900 seconds, the same as the agent timeout, so a trial that uses all +# of it can lose the proxy before the verifier runs. A production app allows +# up to 4 hours. No card is minted, so the host only has to be valid. +[browser.config.params.evervault] +ttl = 900 +hosts = ["checkout-test.adyen.com"] diff --git a/arms/codex-pw-kernel-geo.toml b/arms/codex-pw-kernel-geo.toml new file mode 100644 index 0000000..6054e7d --- /dev/null +++ b/arms/codex-pw-kernel-geo.toml @@ -0,0 +1,18 @@ +# Same as codex-pw-kernel, but each trial gets a new residential proxy in the +# country of the task's site (the task's `country`). +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser] +configurator = "geo" +configurator_args = { proxy = "residential" } + +[browser.config] +provider = "kernel" +params = { stealth = true, timeout_seconds = 1800 } diff --git a/arms/codex-pw-kernel-registry.toml b/arms/codex-pw-kernel-registry.toml new file mode 100644 index 0000000..56dba9b --- /dev/null +++ b/arms/codex-pw-kernel-registry.toml @@ -0,0 +1,20 @@ +# Same as codex-pw-kernel-geo, but each site's browser settings, proxy and +# guidance come from Kernel's config registry for the task's start URL, with +# proxies limited to the task's country. Sites the registry has no +# recommendation for get the geo arm's settings. +agent = "codex" +model = "gpt-6-luna" +agent_version = "0.163.0" +toolset = "playwright-mcp" +environment = "remote-browser" + +[agent_kwargs] +disable_web_search = true + +[browser] +configurator = "registry" +configurator_args = { steering = true } + +[browser.config] +provider = "kernel" +params = { stealth = true, timeout_seconds = 1800 } diff --git a/benchmarks/retail-cart-geo.toml b/benchmarks/retail-cart-geo.toml new file mode 100644 index 0000000..9c2db02 --- /dev/null +++ b/benchmarks/retail-cart-geo.toml @@ -0,0 +1,11 @@ +# The retail-cart tasks reached from each site's own market: a Kernel browser +# with a residential proxy in the site's country, and a Kernel browser sent +# through Evervault's EU relay. Tasks need a `country`; pass a task file with +# `--suite-arg tasks=`. +suite = "retail-cart" +arms = [ + "codex-pw-kernel-geo", + "codex-pw-kernel-evervault-eu", + "codex-pw-kernel-evervault-residential", + "codex-pw-kernel-registry", +] diff --git a/benchmarks/retail-cart-providers.toml b/benchmarks/retail-cart-providers.toml index 61f9841..79c1a82 100644 --- a/benchmarks/retail-cart-providers.toml +++ b/benchmarks/retail-cart-providers.toml @@ -3,6 +3,7 @@ suite = "retail-cart" arms = [ "codex-pw-kernel", + "codex-pw-kernel-evervault", "codex-pw-browserbase", "codex-pw-steel", "codex-pw-hyperbrowser", diff --git a/configurators/geo/configure b/configurators/geo/configure new file mode 100755 index 0000000..b63a0b4 --- /dev/null +++ b/configurators/geo/configure @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Give each trial a proxy in the country of the task's site. + +The country comes from `metadata.country` in the task's task.toml, so a site +is reached from the market it serves. The proxy type is the `proxy` arg +(default `residential`). The proxy is created per trial and deleted after it. + +stdin: {"task": {"config": str, "instruction": str}, "args": {"proxy": str}} +stdout: {"browser": {...}, "steering": null, "provenance": {...}} +""" + +from __future__ import annotations + +import json +import sys +import tomllib + + +def main() -> int: + payload = json.load(sys.stdin) + metadata = tomllib.loads(payload["task"]["config"]).get("metadata", {}) + country = metadata.get("country") + if not country: + raise SystemExit(f"Task for {metadata.get('site')!r} has no metadata.country") + proxy = (payload.get("args") or {}).get("proxy", "residential") + json.dump( + { + "browser": {"proxy_create": {"type": proxy, "config": {"country": country}}}, + "steering": None, + "provenance": {"site": metadata.get("site"), "proxy": proxy, "country": country}, + }, + sys.stdout, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/configurators/registry/configure b/configurators/registry/configure new file mode 100755 index 0000000..856ba1b --- /dev/null +++ b/configurators/registry/configure @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Choose browser settings from Kernel's config registry for the task's own site. + +The task's `metadata.start_url` is looked up directly, so no model has to name +the site, and proxies are limited to the task's `metadata.country`. The +recommendation of the project's latest analysis of that exact target is +preferred, since an analysis started with `resolve` reaches global knowledge +later; otherwise the global recommendation is used. A recommendation's +browser settings and proxy `create` body are used as given. +A site with no recommendation falls back to `fallback`, by default a stealth +browser with a residential proxy in the task's country, so those trials match +the `geo` configurator. The registry's guidance becomes steering when +`steering` is true. + +stdin: {"task": {"config": str, "instruction": str}, "args": {"steering": bool}} +stdout: {"browser": {...}, "steering": str | null, "provenance": {...}} +""" + +from __future__ import annotations + +import json +import sys +import tomllib +from typing import Any + +from kernel import Kernel + +BROWSER_KEYS = ("stealth", "gpu", "headless", "viewport") + + +def project_recommendation(client: Kernel, target: str) -> dict[str, Any] | None: + host = target.split("/")[2] + for summary in client.config_registry.list(search=host): + if summary.target == target and summary.analysis_status == "completed": + analysis = client.config_registry.analyses.retrieve(summary.analysis_id) + recommendation = analysis.model_dump(mode="json", exclude_none=True).get( + "recommendation" + ) + if (recommendation or {}).get("type") == "recommendation": + return recommendation + return None + + +def main() -> int: + payload = json.load(sys.stdin) + args = payload.get("args") or {} + metadata = tomllib.loads(payload["task"]["config"]).get("metadata", {}) + url, country = metadata.get("start_url"), metadata.get("country") + if not url or not country: + raise SystemExit(f"Task for {metadata.get('site')!r} needs metadata.start_url and country") + + client = Kernel() + lookup = client.config_registry.lookup(url=url, allowed_proxy_countries=[country]) + lookup = lookup.model_dump(mode="json", exclude_none=True) + recommendation = ( + project_recommendation(client, lookup["target"]["normalized"]) + or lookup.get("recommendation") + or {} + ) + proxy = (recommendation.get("proxy") or {}).get("create") + if recommendation.get("type") == "recommendation": + recommended = recommendation.get("browser") or {} + browser: dict[str, Any] = { + key: recommended[key] for key in BROWSER_KEYS if key in recommended + } + if proxy: + browser["proxy_create"] = proxy + source = "registry" + else: + browser = { + "stealth": True, + "proxy_create": {"type": "residential", "config": {"country": country}}, + **(args.get("fallback") or {}), + } + source = "fallback" + json.dump( + { + "browser": browser, + "steering": lookup.get("guidance") if args.get("steering", False) else None, + "provenance": { + "source": source, + "matched_target": recommendation.get("matched_target"), + "evidence": recommendation.get("evidence"), + }, + }, + sys.stdout, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/harness/environment.py b/harness/environment.py index df773f3..b18ade4 100644 --- a/harness/environment.py +++ b/harness/environment.py @@ -25,6 +25,9 @@ and drives a browser from any provider in `harness.providers` over CDP, so providers are compared with the same agent, tools and container. Its config file names the `provider` and that provider's session create `params`. +Settings a configurator writes beside them, such as `stealth` or a +`proxy_create` entry, override `params`; only the Kernel providers accept +`proxy_create`. """ from __future__ import annotations @@ -232,7 +235,11 @@ def __init__(self, environment_dir: Path, *args: Any, **kwargs: Any) -> None: config = json.loads((Path(environment_dir) / CONFIG_FILE).read_text()) self._provider_name: str = config["provider"] self._provider = providers.get(self._provider_name) - self._create_params: dict[str, Any] = config.get("params") or {} + # A configurator's settings sit beside `params` and override them. + self._create_params: dict[str, Any] = { + **(config.get("params") or {}), + **{key: value for key, value in config.items() if key not in ("provider", "params")}, + } self._browser_session_id: str | None = None super().__init__(environment_dir, *args, **kwargs) diff --git a/harness/providers.py b/harness/providers.py index 9326bfa..0376031 100644 --- a/harness/providers.py +++ b/harness/providers.py @@ -10,6 +10,7 @@ from __future__ import annotations import os +import uuid from dataclasses import dataclass from typing import Any @@ -34,16 +35,118 @@ async def release(self, session_id: str) -> None: class KernelProvider(Provider): + """A Kernel browser. A `proxy_create` param, a POST /proxies body, creates a + proxy for the session and deletes it after the browser, as in + harness.environment.""" + env = ("KERNEL_API_KEY",) - async def create(self, params: dict[str, Any]) -> Session: - async with AsyncKernel() as client: + def __init__(self) -> None: + self._proxies: dict[str, str] = {} + + async def _create_browser( + self, client: AsyncKernel, params: dict[str, Any], proxy: dict[str, Any] | None + ) -> Session: + from harness.environment import RUN_ID_ENV, proxy_prefix + + if proxy is None: browser = await client.browsers.create(**params) + return Session(browser.session_id, browser.cdp_ws_url) + run_id = os.environ.get(RUN_ID_ENV) or "local" + name = f"{proxy_prefix(run_id)}{uuid.uuid4().hex[:12]}" + kernel_proxy = await client.proxies.create(**proxy, name=name) + try: + browser = await client.browsers.create(**params, proxy_id=kernel_proxy.id) + except Exception: + await client.proxies.delete(kernel_proxy.id) + raise + self._proxies[browser.session_id] = kernel_proxy.id return Session(browser.session_id, browser.cdp_ws_url) + async def create(self, params: dict[str, Any]) -> Session: + params = dict(params) + proxy = params.pop("proxy_create", None) + async with AsyncKernel() as client: + return await self._create_browser(client, params, proxy) + async def release(self, session_id: str) -> None: + # The browser goes first: without its proxy it would reach sites directly. async with AsyncKernel() as client: await client.browsers.delete_by_id(session_id) + proxy_id = self._proxies.pop(session_id, None) + if proxy_id is not None: + await client.proxies.delete(proxy_id) + + +class KernelEvervaultProvider(KernelProvider): + """A Kernel browser whose whole session goes through Evervault's browser proxy. + + Each session mints its own Evervault Browser Tokens session, which carries + the proxy credentials, and creates a Kernel custom proxy from them with + Evervault's CA bundle, since the proxy terminates TLS. The mint holds one + placeholder credential: the session is only used for its proxy, and no + real value is ever revealed. `params` holds the Kernel browser create + params, plus an `evervault` table with the mint's `ttl`, the `hosts` it + may reveal to, optionally `relay_host` to use another region's relay than + the one the mint returns (the credentials work on any), and optionally + `proxy_settings` for Evervault's egress. A residential `proxy_create`, as + the `geo` configurator writes it, becomes Evervault's residential egress in + that country instead of a Kernel proxy. + """ + + env = ("KERNEL_API_KEY", "EV_APP_UUID", "EV_API_KEY") + mint_url = "https://api.evervault.com/browser-tokens" + ca_url = "https://ca.evervault.com" + + def __init__(self) -> None: + super().__init__() + self._ca_bundle: str | None = None + + async def _mint(self, settings: dict[str, Any]) -> dict[str, Any]: + body: dict[str, Any] = { + "ttl": settings.get("ttl", 900), + "data": [{"id": "bench", "category": "credential.password", "value": "bench"}], + "permissions": [{"action": "decrypt", "target": host} for host in settings["hosts"]], + } + if "proxy_settings" in settings: + body["proxySettings"] = settings["proxy_settings"] + async with httpx.AsyncClient(timeout=30) as client: + if self._ca_bundle is None: + response = await client.get(self.ca_url) + response.raise_for_status() + self._ca_bundle = response.text + response = await client.post( + self.mint_url, + json=body, + auth=(os.environ["EV_APP_UUID"], os.environ["EV_API_KEY"]), + ) + # The response body carries the proxy password, so it is never logged. + if response.is_error: + raise RuntimeError(f"Evervault mint failed with HTTP {response.status_code}") + return response.json()["proxyConfiguration"] + + async def create(self, params: dict[str, Any]) -> Session: + browser_params = dict(params) + settings = dict(browser_params.pop("evervault")) + egress = browser_params.pop("proxy_create", None) + if egress is not None: + if egress.get("type") != "residential": + raise ValueError( + f"Evervault egress can only be residential, not {egress.get('type')}" + ) + settings["proxy_settings"] = {"type": "residential", **(egress.get("config") or {})} + proxy = await self._mint(settings) + config = { + "host": settings.get("relay_host") or proxy["hostname"], + "port": proxy["port"], + "username": proxy["username"], + "password": proxy["password"], + "ca_bundle": self._ca_bundle, + } + async with AsyncKernel() as client: + return await self._create_browser( + client, browser_params, {"type": "custom", "protocol": "https", "config": config} + ) class HttpProvider(Provider): @@ -131,6 +234,7 @@ async def release(self, session_id: str) -> None: PROVIDERS: dict[str, Provider] = { "kernel": KernelProvider(), + "kernel-evervault": KernelEvervaultProvider(), "browserbase": BrowserbaseProvider(), "steel": SteelProvider(), "hyperbrowser": HyperbrowserProvider(), diff --git a/suites/retail-cart/suite.py b/suites/retail-cart/suite.py index 879c005..f09e838 100644 --- a/suites/retail-cart/suite.py +++ b/suites/retail-cart/suite.py @@ -13,6 +13,10 @@ itself (`cart_url`, or `cart_api` fetched from the page) and records `cart_verified` when it lists the item (`cart_match`, a case-insensitive regular expression). + +Tasks come from tasks.toml, or from another file of the same shape given as +the `tasks` suite arg. A task's optional `country` is recorded in its +task.toml metadata, where the `geo` configurator reads it. """ from __future__ import annotations @@ -52,6 +56,7 @@ class Task: cart_match: str cart_url: str | None = None cart_api: str | None = None + country: str | None = None timeout_sec: int = 900 @@ -59,12 +64,12 @@ def preflight() -> None: """Nothing to check: tasks need only the Kernel browser Harbor creates.""" -def load_tasks() -> list[Task]: - return [Task(**entry) for entry in tomllib.loads((HERE / "tasks.toml").read_text())["tasks"]] +def load_tasks(path: Path = HERE / "tasks.toml") -> list[Task]: + return [Task(**entry) for entry in tomllib.loads(path.read_text())["tasks"]] -def generate(task_ids: list[str] | None, out_dir: Path) -> list[Path]: - tasks = load_tasks() +def generate(task_ids: list[str] | None, out_dir: Path, tasks: str | None = None) -> list[Path]: + tasks = load_tasks(Path(tasks)) if tasks else load_tasks() if task_ids: known = {task.id: task for task in tasks} missing = [task_id for task_id in task_ids if task_id not in known] @@ -108,6 +113,8 @@ def _write(task: Task, task_dir: Path) -> Path: config["source"] = "retail-cart" config["task"] = {"name": f"retail-cart/{task.id}", "keywords": [task.site]} config["metadata"] = {"site": task.site, "start_url": task.start_url} + if task.country: + config["metadata"]["country"] = task.country config["environment"] = { "build_timeout_sec": 1200.0, "network_mode": "public", diff --git a/tests/test_environment.py b/tests/test_environment.py index 0f0b523..50b1c5c 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -215,7 +215,7 @@ async def release(self, session_id): self.calls.append(("release", session_id)) -def _remote_environment(tmp_path, monkeypatch, provider): +def _remote_environment(tmp_path, monkeypatch, provider, config=None): from harbor.environments.base import ExecResult from harbor.environments.docker.docker import DockerEnvironment @@ -241,7 +241,7 @@ def init(self, environment_dir, **kwargs): monkeypatch.setattr(DockerEnvironment, "exec", exec) monkeypatch.setitem(environment.providers.PROVIDERS, "fake", provider) (tmp_path / "kernel-config.json").write_text( - json.dumps({"provider": "fake", "params": {"stealth": True}}) + json.dumps(config or {"provider": "fake", "params": {"stealth": True}}) ) return environment.RemoteBrowserEnvironment(environment_dir=tmp_path), calls @@ -267,3 +267,22 @@ def test_remote_browser_creation_failure_is_identifiable(tmp_path, monkeypatch): env, _ = _remote_environment(tmp_path, monkeypatch, FakeProvider(fail=True)) with pytest.raises(environment.BrowserCreationError, match="no capacity"): asyncio.run(env.start(force_build=False)) + + +def test_configurator_settings_override_the_remote_browser_params(tmp_path, monkeypatch): + provider = FakeProvider() + proxy = {"type": "isp", "config": {"country": "GB"}} + config = { + "provider": "fake", + "params": {"stealth": True, "timeout_seconds": 1800}, + "stealth": False, + "proxy_create": proxy, + } + env, _ = _remote_environment(tmp_path, monkeypatch, provider, config) + + asyncio.run(env.start(force_build=False)) + + assert provider.calls[0] == ( + "create", + {"stealth": False, "timeout_seconds": 1800, "proxy_create": proxy}, + ) diff --git a/tests/test_geo.py b/tests/test_geo.py new file mode 100644 index 0000000..4f6d10b --- /dev/null +++ b/tests/test_geo.py @@ -0,0 +1,30 @@ +import json +import subprocess + +from harness.definitions import ROOT + +CONFIGURE = ROOT / "configurators/geo/configure" + + +def _run(config, args=None): + payload = {"task": {"config": config, "instruction": ""}, "args": args or {}} + return subprocess.run( + [CONFIGURE], input=json.dumps(payload), capture_output=True, text=True, check=False + ) + + +def test_the_proxy_is_in_the_task_country(): + result = _run('[metadata]\nsite = "shop.example"\ncountry = "PL"\n', {"proxy": "isp"}) + output = json.loads(result.stdout) + assert output["browser"] == {"proxy_create": {"type": "isp", "config": {"country": "PL"}}} + + +def test_a_task_without_a_country_is_refused(): + result = _run('[metadata]\nsite = "shop.example"\n') + assert result.returncode != 0 + assert "country" in result.stderr + + +def test_the_default_proxy_is_residential(): + output = json.loads(_run('[metadata]\ncountry = "GB"\n').stdout) + assert output["browser"]["proxy_create"]["type"] == "residential" diff --git a/tests/test_providers.py b/tests/test_providers.py index 9a9cbdf..a7d67f5 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -84,3 +84,149 @@ def test_failed_create_raises(monkeypatch): def test_unknown_provider_is_refused(): with pytest.raises(SystemExit, match="nope"): providers.get("nope") + + +class _FakeKernel: + def __init__(self, calls): + self.calls = calls + self.proxies = self + self.browsers = self + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def create(self, **kwargs): + if "type" in kwargs: + self.calls.append(("proxy.create", kwargs)) + return type("Proxy", (), {"id": "p1"}) + self.calls.append(("browser.create", kwargs)) + return type("Browser", (), {"session_id": "b1", "cdp_ws_url": "wss://k/1"}) + + async def delete(self, proxy_id): + self.calls.append(("proxy.delete", proxy_id)) + + async def delete_by_id(self, session_id): + self.calls.append(("browser.delete", session_id)) + + +def test_kernel_evervault_proxies_the_browser_through_a_minted_session(monkeypatch): + monkeypatch.setenv("EV_APP_UUID", "app_1") + monkeypatch.setenv("EV_API_KEY", "ev:key:1") + monkeypatch.setenv("BENCH_RUN_ID", "r1") + requests = [] + + def handler(request): + requests.append(request) + if request.url.host == "ca.evervault.com": + return httpx.Response(200, text="PEM") + proxy = {"hostname": "us-va.browser.relay.evervault.app", "port": 443} + return httpx.Response( + 200, json={"proxyConfiguration": {**proxy, "username": "app_1", "password": "jwe"}} + ) + + real_client = httpx.AsyncClient + monkeypatch.setattr( + providers.httpx, + "AsyncClient", + lambda **kwargs: real_client(transport=httpx.MockTransport(handler), **kwargs), + ) + calls = [] + monkeypatch.setattr(providers, "AsyncKernel", lambda: _FakeKernel(calls)) + provider = providers.KernelEvervaultProvider() + + session = asyncio.run( + provider.create({"stealth": True, "evervault": {"ttl": 1800, "hosts": ["a.example"]}}) + ) + asyncio.run(provider.release(session.id)) + + assert session == providers.Session("b1", "wss://k/1") + mint = json.loads(requests[1].content) + assert mint["ttl"] == 1800 + assert mint["permissions"] == [{"action": "decrypt", "target": "a.example"}] + proxy = calls[0][1] + assert proxy["type"] == "custom" and proxy["protocol"] == "https" + assert proxy["name"].startswith("bench-r1-") + assert proxy["config"] == { + "host": "us-va.browser.relay.evervault.app", + "port": 443, + "username": "app_1", + "password": "jwe", + "ca_bundle": "PEM", + } + assert calls[1] == ("browser.create", {"stealth": True, "proxy_id": "p1"}) + assert calls[2:] == [("browser.delete", "b1"), ("proxy.delete", "p1")] + + +def test_kernel_evervault_mint_failure_does_not_leak_the_body(monkeypatch): + monkeypatch.setenv("EV_APP_UUID", "app_1") + monkeypatch.setenv("EV_API_KEY", "ev:key:1") + + def handler(request): + if request.url.host == "ca.evervault.com": + return httpx.Response(200, text="PEM") + return httpx.Response(403, text="secret detail") + + real_client = httpx.AsyncClient + monkeypatch.setattr( + providers.httpx, + "AsyncClient", + lambda **kwargs: real_client(transport=httpx.MockTransport(handler), **kwargs), + ) + with pytest.raises(RuntimeError, match="HTTP 403") as error: + asyncio.run(providers.KernelEvervaultProvider().create({"evervault": {"hosts": ["a"]}})) + assert "secret" not in str(error.value) + + +def test_kernel_creates_and_deletes_a_proxy_per_session(monkeypatch): + calls = [] + monkeypatch.setattr(providers, "AsyncKernel", lambda: _FakeKernel(calls)) + provider = providers.KernelProvider() + proxy = {"type": "residential", "config": {"country": "GB"}} + + session = asyncio.run(provider.create({"stealth": True, "proxy_create": proxy})) + asyncio.run(provider.release(session.id)) + + assert calls[0][0] == "proxy.create" and calls[0][1]["config"] == {"country": "GB"} + assert calls[1] == ("browser.create", {"stealth": True, "proxy_id": "p1"}) + assert calls[2:] == [("browser.delete", "b1"), ("proxy.delete", "p1")] + + +def test_kernel_evervault_turns_a_residential_proxy_into_evervault_egress(monkeypatch): + monkeypatch.setenv("EV_APP_UUID", "app_1") + monkeypatch.setenv("EV_API_KEY", "ev:key:1") + requests = [] + + def handler(request): + requests.append(request) + if request.url.host == "ca.evervault.com": + return httpx.Response(200, text="PEM") + proxy = {"hostname": "us-va.browser.relay.evervault.app", "port": 443} + return httpx.Response( + 200, json={"proxyConfiguration": {**proxy, "username": "app_1", "password": "jwe"}} + ) + + real_client = httpx.AsyncClient + monkeypatch.setattr( + providers.httpx, + "AsyncClient", + lambda **kwargs: real_client(transport=httpx.MockTransport(handler), **kwargs), + ) + calls = [] + monkeypatch.setattr(providers, "AsyncKernel", lambda: _FakeKernel(calls)) + + asyncio.run( + providers.KernelEvervaultProvider().create( + { + "evervault": {"hosts": ["a.example"]}, + "proxy_create": {"type": "residential", "config": {"country": "GB"}}, + } + ) + ) + + mint = json.loads(requests[1].content) + assert mint["proxySettings"] == {"type": "residential", "country": "GB"} + assert calls[0][1]["type"] == "custom" + assert "proxy_create" not in calls[1][1] diff --git a/tests/test_registry.py b/tests/test_registry.py new file mode 100644 index 0000000..eea01e2 --- /dev/null +++ b/tests/test_registry.py @@ -0,0 +1,120 @@ +import json +import os +import subprocess + +from harness.definitions import ROOT + +CONFIGURE = ROOT / "configurators/registry/configure" + +FAKE_KERNEL = """ +import json, os + + +class _Lookup: + def __init__(self, data): + self.data = data + + def model_dump(self, **kwargs): + return self.data + + +class _Summary: + def __init__(self, data): + self.__dict__.update(data) + + +class _Analyses: + def retrieve(self, analysis_id): + return _Lookup({"recommendation": json.loads(os.environ["FAKE_ANALYSIS"])}) + + +class _Registry: + analyses = _Analyses() + + def list(self, search): + return [_Summary(item) for item in json.loads(os.environ["FAKE_PROJECT"])] + + def lookup(self, url, allowed_proxy_countries): + with open(os.environ["FAKE_LOOKUP_LOG"], "w") as log: + json.dump({"url": url, "countries": allowed_proxy_countries}, log) + return _Lookup(json.loads(os.environ["FAKE_LOOKUP"])) + + +class Kernel: + config_registry = _Registry() +""" + +TASK = '[metadata]\nsite = "shop.example"\nstart_url = "https://shop.example/"\ncountry = "GB"\n' + + +TARGET = {"target": {"normalized": "https://shop.example/"}} + + +def _run(tmp_path, lookup, args=None, project=(), analysis=None): + (tmp_path / "kernel.py").write_text(FAKE_KERNEL) + env = { + **os.environ, + "PYTHONPATH": str(tmp_path), + "FAKE_LOOKUP": json.dumps({**TARGET, **lookup}), + "FAKE_PROJECT": json.dumps(list(project)), + "FAKE_ANALYSIS": json.dumps(analysis), + "FAKE_LOOKUP_LOG": str(tmp_path / "lookup.json"), + } + payload = {"task": {"config": TASK, "instruction": ""}, "args": args or {}} + result = subprocess.run( + [CONFIGURE], input=json.dumps(payload), capture_output=True, text=True, env=env, check=True + ) + return json.loads(result.stdout) + + +def test_a_recommendation_is_used_as_given(tmp_path): + proxy = {"type": "isp", "config": {"country": "GB"}} + lookup = { + "recommendation": { + "type": "recommendation", + "browser": {"stealth": False, "gpu": False, "headless": False}, + "proxy": {"mode": "managed", "create": proxy}, + }, + "guidance": "Use the search box.", + } + output = _run(tmp_path, lookup, {"steering": True}) + + assert output["browser"] == { + "stealth": False, + "gpu": False, + "headless": False, + "proxy_create": proxy, + } + assert output["steering"] == "Use the search box." + assert output["provenance"]["source"] == "registry" + log = json.loads((tmp_path / "lookup.json").read_text()) + assert log == {"url": "https://shop.example/", "countries": ["GB"]} + + +def test_a_site_without_a_recommendation_gets_the_geo_settings(tmp_path): + output = _run(tmp_path, {"working_configurations": []}) + + assert output["browser"] == { + "stealth": True, + "proxy_create": {"type": "residential", "config": {"country": "GB"}}, + } + assert output["steering"] is None + assert output["provenance"]["source"] == "fallback" + + +def test_the_project_analysis_is_preferred_over_global_knowledge(tmp_path): + analysis = { + "type": "recommendation", + "browser": {"stealth": False}, + "proxy": {"create": {"type": "isp", "config": {"country": "GB"}}}, + } + project = [ + {"target": "https://shop.example/", "analysis_status": "completed", "analysis_id": "a1"} + ] + output = _run(tmp_path, {}, project=project, analysis=analysis) + + assert output["browser"] == { + "stealth": False, + "proxy_create": {"type": "isp", "config": {"country": "GB"}}, + } + assert output["provenance"]["source"] == "registry" diff --git a/tests/test_retail_cart.py b/tests/test_retail_cart.py index 962c9a5..b80d626 100644 --- a/tests/test_retail_cart.py +++ b/tests/test_retail_cart.py @@ -32,6 +32,18 @@ def test_generate_writes_a_harbor_task(tmp_path): assert (task_dir / "environment/bench/finalize-mcp").is_file() +def test_a_tasks_file_and_country_can_be_given(tmp_path): + tasks = tmp_path / "tasks.toml" + tasks.write_text( + '[[tasks]]\nid = "shop"\nsite = "shop.example"\nstart_url = "https://shop.example/"\n' + 'item = "a pair of socks"\ncart_url = "https://shop.example/cart"\n' + 'cart_match = "sock"\ncountry = "GB"\n' + ) + [task_dir] = suite.generate(None, tmp_path / "out", tasks=str(tasks)) + + assert tomllib.loads((task_dir / "task.toml").read_text())["metadata"]["country"] == "GB" + + def test_unknown_task_is_refused(tmp_path): with pytest.raises(SystemExit, match="nope"): suite.generate(["nope"], tmp_path)