From 4f678a395193a636d7840bfeae71fb0f08e2c1d5 Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Thu, 19 Nov 2026 11:25:40 +0100 Subject: [PATCH 1/6] feat(builder): choose hosts or Boat sandboxes on the Where step Machines are Hosts (this machine and Settings hosts, one shard each) or Sandboxes (a Boat VM per unit, as many at once as the account's limits say), never both. Sandboxes offer the sizes the account's limits list; a size without more memory than a unit's own VM is refused, read from a Where option's new needs..vm (OSWorld1 and OSWorld2: 4 GB). How it runs is fixed in a sandbox and for a benchmark whose units boot their own VM (no container: it cannot start them), with a single VM provider on the same line. A setup or a check on Boat no longer falls back silently to local processes: Review and run.Run refuse it, as they refuse a computer-port agent. Boat's limits are read once a minute for both sizes and slots. --- .../benchmarks/osworld1/standard.json | 5 +- .../benchmarks/osworld2/standard.json | 5 +- app/nullius/standard/boat.py | 33 ++++++-- app/nullius/standard/definitions.py | 23 +++++- app/nullius/standard/recipe.py | 30 ++++++-- app/nullius/standard/run.py | 4 +- app/static/builder/command-builder.css | 3 +- app/static/builder/command-builder.js | 77 +++++++++++++------ app/tests/catalogue_docs.py | 3 +- app/tests/test_standard_bench_options.py | 8 ++ app/tests/test_standard_boat.py | 58 +++++++++++--- app/ui_tests/test_agents.py | 18 ++++- app/ui_tests/test_builder_where.py | 75 ++++++++++++++---- app/web/builder.py | 14 ++-- app/web/execute.py | 24 ++++-- docs/benchmarks/supported/osworld1.md | 2 +- docs/benchmarks/supported/osworld2.md | 2 +- docs/builder.md | 2 +- docs/standard.md | 9 ++- 19 files changed, 306 insertions(+), 89 deletions(-) diff --git a/app/nullius/catalogue/benchmarks/osworld1/standard.json b/app/nullius/catalogue/benchmarks/osworld1/standard.json index 0b588a1e..c460a375 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/standard.json +++ b/app/nullius/catalogue/benchmarks/osworld1/standard.json @@ -253,7 +253,10 @@ "docker": { "host": [ "docker" - ] + ], + "vm": { + "memory_gb": 4 + } } } }, diff --git a/app/nullius/catalogue/benchmarks/osworld2/standard.json b/app/nullius/catalogue/benchmarks/osworld2/standard.json index 55930670..5378d2a7 100644 --- a/app/nullius/catalogue/benchmarks/osworld2/standard.json +++ b/app/nullius/catalogue/benchmarks/osworld2/standard.json @@ -192,7 +192,10 @@ "host": [ "docker", "scoring_programs" - ] + ], + "vm": { + "memory_gb": 4 + } } } }, diff --git a/app/nullius/standard/boat.py b/app/nullius/standard/boat.py index 1eb6fe8a..e2778140 100644 --- a/app/nullius/standard/boat.py +++ b/app/nullius/standard/boat.py @@ -56,8 +56,10 @@ API = "https://boat.dev/api/v1" STORE, NAME = "sandbox-provider-boat", "BOAT_API_KEY" -# the VM runs the agent's process, never its model: the smallest is enough -SIZES = {"small": "2 vCPU, 4 GB", "default": "4 vCPU, 8 GB", "large": "", "xlarge": ""} +# Boat's machine types, by name: its label and, where measured, its vCPUs and memory. The VM runs the agent's +# process, never its model: the smallest is enough, unless a task boots a VM of its own inside it (fits). +SIZES = {"small": {"label": "Small", "cpus": 2, "memory_gb": 4}, "default": {"label": "Default", "cpus": 4, "memory_gb": 8}, + "large": {"label": "Large"}, "xlarge": {"label": "XLarge"}} SIZE = "small" TTL = 2 * 3600 # a VM's auto-stop at least, and a free trial's at most USER = "agent" @@ -153,10 +155,11 @@ def said(d: dict) -> str: return str(d.get("message") or d.get("code") or d.get("status") or "no answer") -def slots(client: Client) -> Optional[int]: - """How many VMs the account may run at once; None when its API does not say.""" - n = client.limits().get("maxActiveSandboxes") - return n if isinstance(n, int) and n > 0 else None +def slots(client: Client, limits: Optional[dict] = None) -> Optional[int]: + """How many VMs the account may run at once; None when its API does not say. ``limits``: its + answer to ``GET /limits`` when already asked.""" + n = (client.limits() if limits is None else limits).get("maxActiveSandboxes") + return n if isinstance(n, int) and not isinstance(n, bool) and n > 0 else None def starts_problem(client: Client, n: int) -> Optional[str]: @@ -172,16 +175,30 @@ def starts_problem(client: Client, n: int) -> Optional[str]: return None -def machine_types(client: Client) -> Dict[str, Optional[str]]: +def machine_types(client: Client, limits: Optional[dict] = None) -> Dict[str, Optional[str]]: """Each machine type the account knows: None when it may start one, else boat's reason why not ("not during the free trial"). Empty when its API does not say.""" out = {} - for t in client.limits().get("machineTypes") or []: + for t in (client.limits() if limits is None else limits).get("machineTypes") or []: if isinstance(t, dict) and isinstance(t.get("type"), str): out[t["type"]] = None if t.get("available") else str(t.get("reason") or "not available on this account") return out +def spec(size: str) -> str: + """A size's vCPUs and memory as a card says them, or "" when unmeasured.""" + z = SIZES.get(size) or {} + return f"{z['cpus']} vCPU, {z['memory_gb']} GB" if "cpus" in z and "memory_gb" in z else "" + + +def fits(size: str, vm_memory_gb) -> bool: + """Whether a VM of ``size`` can boot a task's own VM of ``vm_memory_gb`` (OSWorld's guest): the guest's + memory must be free inside it, so the size needs more than the guest; a size of unmeasured memory, or + no guest, is not refused here.""" + have = (SIZES.get(size) or {}).get("memory_gb") + return not vm_memory_gb or not isinstance(have, (int, float)) or have > vm_memory_gb + + def size_problem(client: Client, size: str) -> Optional[str]: """Why this account cannot start a VM of ``size``, or None.""" if size not in SIZES: diff --git a/app/nullius/standard/definitions.py b/app/nullius/standard/definitions.py index 76b84d7e..5ac10791 100644 --- a/app/nullius/standard/definitions.py +++ b/app/nullius/standard/definitions.py @@ -517,9 +517,13 @@ def _bench_options(d: dict) -> List[str]: bad.append(f"{w}.needs: value -> {{embeddings, reranker, host, files}}") continue for v, n in needs.items(): - if option_problem(o, v) is not None or not isinstance(n, dict) or any(k not in NEED_KEYS + ("when",) for k in n): - bad.append(f"{w}.needs.{v}: one of its values -> {{embeddings: a model, reranker: a model, host: [requirements], files: [names]}}") + if option_problem(o, v) is not None or not isinstance(n, dict) or any(k not in NEED_KEYS + ("when", "vm") for k in n): + bad.append(f"{w}.needs.{v}: one of its values -> {{embeddings: a model, reranker: a model, host: [requirements], files: [names], vm: {{memory_gb}}}}") continue + if "vm" in n and (o.get("step") != "where" or not isinstance(n["vm"], dict) or set(n["vm"]) != {"memory_gb"} + or isinstance(n["vm"]["memory_gb"], bool) or not isinstance(n["vm"]["memory_gb"], (int, float)) + or n["vm"]["memory_gb"] <= 0): + bad.append(f"{w}.needs.{v}.vm: on a Where option, the memory in GB of the VM each unit boots: {{memory_gb: a number}}") if not _when_valid(name, n.get("when") or {}, opts): bad.append(f"{w}.needs.{v}.when: another option -> the values that need these requirements") for kind in ("embeddings", "reranker"): @@ -625,6 +629,21 @@ def option_needs(bench: dict, values: dict) -> dict: return out +def vm_memory_gb(bench: dict, values: dict) -> float: + """The memory in GB of the VM each unit boots of its own (OSWorld's guest) under the chosen option + values: a Where option's ``needs..vm``; 0 when it boots none.""" + opts = bench.get("options") or {} + return max([((opts.get(n, {}).get("needs") or {}).get(v) or {}).get("vm", {}).get("memory_gb", 0) + for n, v in values.items() if _when_applies(opts.get(n, {}).get("when") or {}, opts, values)] or [0]) + + +def boots_vms(bench: dict) -> bool: + """Whether each unit boots a VM of its own on the machine its driver runs on: a Where option of the + benchmark has a value that declares one (``needs..vm``).""" + return any("vm" in (n or {}) for o in (bench.get("options") or {}).values() if o.get("step") == "where" + for n in (o.get("needs") or {}).values()) + + def options_of(d: dict, given: Optional[dict] = None) -> Tuple[dict, List[str]]: """Every option's value for one run - the ones ``given``, the rest at their defaults - and what is wrong with the given ones (a name it does not have, diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 85b62fa9..616678e7 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -547,7 +547,8 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, limit = recipe.get("limit") if limit not in ("", None) and _int(limit, 1) is None: add("limits", "The number of units is a whole number of at least 1, or empty for all of them.") - conc = _int(recipe.get("concurrency") or 1, 1, MAX_CONCURRENT) # empty: one at a time + # empty: one at a time; on Boat, as many as the account runs at once (the run caps it to that, boat.slots) + conc = _int(recipe.get("concurrency") or (MAX_CONCURRENT if backend == "boat" else 1), 1, MAX_CONCURRENT) if conc is None: add("limits", f"Units at a time: a whole number from 1 to {MAX_CONCURRENT}.") targets, why_not = where_of(recipe.get("where", "local")) @@ -557,7 +558,9 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, why = docker_unusable() if why: add("where", f"The container backend cannot run on this machine: {why}. Choose local processes, or a host with Docker or Podman.") - elif backend == "boat" and boat_unusable is not None and (why := boat_unusable("local" in targets)): + elif backend == "boat" and targets != ["local"]: + add("where", "Hosts and Boat sandboxes do not mix: choose Hosts, or Sandboxes alone.") + elif backend == "boat" and boat_unusable is not None and (why := boat_unusable(True)): add("where", f"Boat cannot run: {why}.") elif backend == "boat" and boat_size_problem is not None and str(recipe.get("boat_size") or boat.SIZE) in boat.SIZES: why = boat_size_problem(str(recipe.get("boat_size") or boat.SIZE)) @@ -567,6 +570,20 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, if backend == "boat" and boat_size not in boat.SIZES: add("where", f"Choose a Boat size: {', '.join(boat.SIZES)}.") boat_size = boat.SIZE + # a benchmark whose units boot a VM of their own (OSWorld's guest): the VM starts on the machine the + # driver runs on, which a container cannot do, and on Boat inside a VM big enough to hold it + if b and definitions.boots_vms(b) and backend == "container": + add("where", f"{b['label']} starts each unit's VM on the machine: choose local processes.") + guest = max([definitions.vm_memory_gb(b, v) for v in per_ds.values()] or [0]) + if backend == "boat" and not boat.fits(boat_size, guest): + add("where", f"A {boat.SIZES[boat_size]['label']} VM is too small for {b['label']}, whose units boot a {guest:g} GB VM: choose a larger size.") + # on Boat each unit's agent or capability runs in its VM (run.Run refuses the rest): a setup or a check runs + # on a machine, and the computer port's socket is on this one, out of a VM's reach + computer = "agent.computer/v1" in (b.get("needs") or []) + elsewhere = [definitions.setup_label(b, "original") if p.get("setup") else p["o"]["label"] + for p in picks if p["control"] or p.get("setup") or (computer and side == "agent")] + if backend == "boat" and elsewhere: + add("where", f"{', '.join(elsewhere)} cannot run in a Boat VM: choose Hosts.") # a job that calls no model (a control of a benchmark with no judge) spends nothing and takes no share spends = lambda p: not (p["control"] and not (b.get("score") or {}).get("judge") and not needs["embeddings"] and not needs["reranker"]) paying = len(dss) * sum(1 for p in picks if spends(p)) * len(targets or [None]) @@ -766,7 +783,8 @@ def row(p: dict) -> list: + (f" · judge {judge_model or b['score']['judge']['model']}" + ("" if judge_model else " (official)") if (b.get("score") or {}).get("judge") else "") + "".join(f" · {SERVED_LABEL[r_]} served by {', '.join(s)}" for r_, s in served.items())], [STEP["limits"], "Scope", f"{'first ' + str(n) if isinstance(limit, int) else 'all ' + str(total or '')} {nouns}" - + (" of each shard" if of > 1 and isinstance(limit, int) else "") + f" · {conc} at a time".replace(" ", " ") + + (" of each shard" if of > 1 and isinstance(limit, int) else "") + + (" · as many at a time as Boat allows" if backend == "boat" and not recipe.get("concurrency") else f" · {conc} at a time") + (" of each dataset" if many and isinstance(limit, int) else "") + (f" · {len(pairs)} jobs, one after the other" if len(pairs) > 1 else "") + (f" · {repeat} attempts each (pass^1 to pass^{repeat})" if repeat > 1 else "") @@ -779,9 +797,9 @@ def row(p: dict) -> list: *([[STEP["limits"], "Time limits", " · ".join(f"{k} {v / 60:g} min" for k, v in forced.items())]] if forced else []), *([[STEP["limits"], "Turns per conversation", str(turns)]] if turns else []), *([[STEP["limits"], f"Model calls per {noun}", str(calls)]] if calls else []), - [STEP["where"], "Where", (f"{of} shards, one per machine: " if of > 1 else "") - + ", ".join(where_label(t) for t in targets) + f" · {backend} backend" - + (f" · {boat_size} VMs" if backend == "boat" else "") + [STEP["where"], "Where", (f"Boat sandboxes · one {boat.SIZES[boat_size]['label']} VM per {noun}" if backend == "boat" + else (f"{of} shards, one per machine: " if of > 1 else "") + + ", ".join(where_label(t) for t in targets) + f" · {backend} backend") + "".join(f" · {w.get('label') or k.replace('_', ' ').capitalize()} {(w.get('names') or {}).get(v) or v}" for k, v in {k: v for vals in per_ds.values() for k, v in vals.items()}.items() if (w := (b.get("options") or {})[k]).get("step") == "where")]] if agents_local: diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 9f253f43..09be66b4 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -536,7 +536,9 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas if backend not in BACKENDS: raise Refused(f"backend {backend!r} is not one of {list(BACKENDS)}") if backend == "boat" and other.get("_kind") not in ("agent", "capability"): - backend = "local" # a check or a setup: nothing of an agent's runs, so nothing goes to a VM + # a setup or a check runs on the machine; on Boat it would run here unasked, beside the VMs + raise Refused(f"{other['label']} runs on a machine, not in a Boat VM: run it on this machine or a host " + "(--backend local or container)") if backend == "boat" and other.get("_kind") == "agent" and "agent.computer/v1" in bench.get("needs", []): raise BenchmarkRefused("the computer port uses a socket on this machine, which an agent in a Boat VM " "cannot reach: choose local processes or the container backend") diff --git a/app/static/builder/command-builder.css b/app/static/builder/command-builder.css index 8fd7695d..e7726b99 100644 --- a/app/static/builder/command-builder.css +++ b/app/static/builder/command-builder.css @@ -250,7 +250,8 @@ #nullius-builder .std-backends .nb-tile-note { font-size: 11px; padding-left: 30px; } #nullius-builder .std-boat-sizes { grid-template-columns: repeat(4, minmax(0, 160px)); grid-auto-rows: auto; } #nullius-builder .std-boat-sizes .nb-tile { flex-direction: column; align-items: stretch; gap: 2px; } -#nullius-builder .std-boat-sizes .nb-tile-note { white-space: normal; } +#nullius-builder .std-boat-sizes .nb-tile-note { white-space: normal; padding-left: 0; } +#nullius-builder .std-boat-sizes .nb-tile:not([disabled]) .nb-tile-note { color: var(--nb-muted); } @media (max-width: 760px) { #nullius-builder .std-boat-sizes { grid-template-columns: repeat(2, minmax(0, 180px)); } } @media (max-width: 560px) { #nullius-builder .std-backends { grid-template-columns: minmax(0, 320px); } #nullius-builder .std-boat-sizes { grid-template-columns: repeat(2, minmax(0, 1fr)); } } /* Served by: a row after its model's row, as that row is: the label left, the picker right, the providers chosen diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index 6b8f81d0..fc7b4612 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -456,42 +456,68 @@ return `${esc(name)}${proofSpan(p)}`; }; // The yes to an agent running as you: asked only for the local backend, where it is no sandbox. const consentBox = rec => ``; - // The standard's Where step: one machine or several (one shard each), and the backend on them. + // A benchmark whose units each boot a VM of their own (OSWorld's guest): a Where option has a value declaring + // one (needs..vm); vmMemory, that VM's memory in GB under the chosen values, 0 for none. + const bootsVms = b => Object.values((b || {}).options || {}).some(o => o.step === 'where' && Object.values(o.needs || {}).some(n => n && n.vm)); + const vmMemory = (rec, b) => Math.max(0, ...Object.entries((b || {}).options || {}).filter(([, o]) => o.step === 'where') + .map(([n, o]) => (((o.needs || {})[(rec.bench_options || {})[n] ?? o.default] || {}).vm || {}).memory_gb || 0)); + // Boat's sizes as the account offers them (nullius/standard/boat.py, SIZES), each with why it cannot be chosen: + // the account cannot start one, or it is too small to boot the unit's own VM (boat.fits). + const boatSizes = (rec, b) => { const mem = vmMemory(rec, b); + return ((schema.standardHere || {}).boatSizes || []).map(z => z.why ? {...z, note: 'Not on this account', why: 'Not on this account: ' + z.why} + : mem && typeof z.memory_gb === 'number' && z.memory_gb <= mem ? {...z, note: 'Too small', why: `Too small for ${b.label}'s ${mem} GB VM`} : {...z, why: ''}); }; + const BOAT_MARK = ''; + // The standard's Where step. Machines: hosts (this machine, Settings hosts; one shard each) or Boat sandboxes (a VM + // per unit, as many at once as the account runs), never both. How it runs: on hosts, local processes or a container; + // in a sandbox, or for a benchmark whose units boot VMs of their own (which a container cannot start), fixed. function standardWhere(rec) { const here = schema.standardHere || {}, list = C.whereList(rec), H = schema.hosts || [], b = C.standardBench(rec); + const sandboxes = rec.backend === 'boat', vms = bootsVms(b), noun = unitsOf(rec).noun; + if (vms && rec.backend === 'container') rec.backend = 'local'; // recipe.py refuses it: nothing to choose const backend = rec.backend || 'local'; const selected = C.standardOthers(rec).filter(o => C.chosenOf(rec).includes(o.key)); const dockerWhy = list.includes('local') && here.docker ? 'Not on this machine: ' + here.docker : ''; // Review's rule (nullius/standard/recipe.py): a setup's image installs what has an install; nothing else reaches its container const cannot = selected.some(o => o.setup) ? optionNeeds(rec).host.filter(n => (b.host || {})[n] && !b.host[n].install.length) : []; const containerWhy = dockerWhy || (cannot.length ? cannot.join(', ') + ' cannot be provided in a container: choose local processes.' : ''); - // boat: started where the run runs, with this machine's key (Settings → Sandbox); a host's programs are its prerequisites - const boatWhy = (list.includes('local') ? here.boat : here.boatKey) || ''; const agentRun = b && b.family === 'agent' && selected.some(o => !o.control && !o.setup); + const kind = (v, t, mark, meta, tip, why) => tile({attr: 'std-machines', key: v, on: (v === 'boat') === sandboxes && !why, disabled: !!why, why, + note: why ? 'Not usable here' : '', mark, title: t, meta, tip}); const machine = (v, t, sub) => machineTile('std-where', v, t, sub, list.includes(v)); - const how = (v, t, sub, why) => tile({attr: 'std-backend', key: v, on: backend === v && !why, disabled: !!why, why, note: why ? 'Not usable here' : '', mark: v === 'local' ? icon('terminal') : v === 'boat' ? '' : icon('container'), title: t, tip: sub}); + const how = (v, t, sub, why) => tile({attr: 'std-backend', key: v, on: backend === v && !why, disabled: !!why, why, note: why ? 'Not usable here' : '', mark: v === 'local' ? icon('terminal') : icon('container'), title: t, tip: sub}); const dockerOnly = Object.entries((b || {}).environments || {}).filter(([, e]) => !e.local_ok).map(([n]) => n); const web = agentRun && Object.values((b || {}).environments || {}).some(e => e.network); // an agent's public web proxy + // a VM benchmark's provider of one value goes on the fixed line; several are its cards (benchOptions) + const provider = vms && Object.values(b.options || {}).find(o => o.step === 'where' && o.values && o.values.length === 1 && Object.values(o.needs || {}).some(n => n && n.vm)); + // what a sandbox cannot run (recipe.py): a setup or a check runs on a machine; the computer port's socket is on this one + const outside = sandboxes ? selected.filter(o => o.setup || o.control || ((b.needs || []).includes('agent.computer/v1') && b.family === 'agent')).map(o => o.label) : []; + let sizes = ''; + if (sandboxes && !here.boat) { + const offered = boatSizes(rec, b), fit = offered.find(z => !z.why); + if (fit && !offered.some(z => z.key === (rec.boat_size || 'small') && !z.why)) rec.boat_size = fit.key; + const size = rec.boat_size || 'small'; + sizes = '

VM size

' + + `
${offered.map(z => tile({attr: 'std-boat-size', key: z.key, on: size === z.key && !z.why, + disabled: !!z.why, why: z.why, note: z.note || z.spec, title: z.label})).join('')}
`; + } return '

Where does it run?

' - + `
${machine('local', 'This machine', 'Nullius runs it here, in runs/' + rec.name + '.')}${H.map(h => machine(h.entry, h.label, 'Over ssh: the standard\'s code at this commit is sent there, uv put there if missing; its job comes back as it runs.')).join('')}
` - + (H.length ? (list.some(w => w !== 'local') ? button('Test connection', 'id="nb-test-host"') + ' ' : '') : `

Hosts are named in ${button('Settings → Hosts', 'data-href="/settings#HOSTS"', 'nb-inlinebutton')}; this machine is the only choice until then.

`) - + (b ? benchOptions(rec, b, 'where') : '') + '

How it runs there

' - + `
${how('local', 'Local processes', 'Its commands run as you, not sandboxed.', '')}` - + how('container', 'Container', web ? 'Sandboxed: each unit in a container whose only ways out are its relay and a public web proxy.' - : 'Sandboxed: each unit in a container whose only way out is its relay.', containerWhy) - + how('boat', 'Boat', 'Sandboxed: each unit in a Boat VM of its own, whose only way out is its relay.', boatWhy) + '
' - + (backend === 'boat' && !boatWhy ? boatSizes(rec) : '') + + `
${kind('hosts', 'Hosts', icon('machine'), H.length ? `This machine + ${H.length} ${H.length > 1 ? 'hosts' : 'host'}` : 'This machine', 'One shard per machine.', '')}` + + kind('boat', 'Sandboxes', BOAT_MARK, here.boatSlots ? `Boat · ${here.boatSlots} at once` : 'Boat', `One Boat VM per ${noun}.`, here.boat || '') + '
' + + (sandboxes ? sizes + : '

Machines

' + + `
${machine('local', 'This machine', 'Nullius runs it here, in runs/' + rec.name + '.')}${H.map(h => machine(h.entry, h.label, 'Over ssh: the standard\'s code at this commit is sent there, uv put there if missing; its job comes back as it runs.')).join('')}
` + + (H.length ? (list.some(w => w !== 'local') ? button('Test connection', 'id="nb-test-host"') + ' ' : '') : `

Hosts are named in ${button('Settings → Hosts', 'data-href="/settings#HOSTS"', 'nb-inlinebutton')}.

`)) + + '

How it runs

' + + (sandboxes || vms + ? `

Each ${esc(noun)} in its own VM${provider ? ` · ${esc(provider.label)}: ${esc((provider.names || {})[provider.values[0]] || words(provider.values[0]))}` : ''}

` + : `
${how('local', 'Local processes', 'Its commands run as you, not sandboxed.', '')}` + + how('container', 'Container', web ? 'Sandboxed: each unit in a container whose only ways out are its relay and a public web proxy.' + : 'Sandboxed: each unit in a container whose only way out is its relay.', containerWhy) + '
') + + (b ? benchOptions(rec, b, 'where') : '') + (agentRun && backend === 'local' ? consentBox(rec) : '') - + (dockerOnly.length && backend !== 'container' && agentRun ? `

${esc(dockerOnly.join(', '))}: ${dockerOnly.length > 1 ? 'these environments need' : 'this environment needs'} a container - on ${backend === 'boat' ? 'Boat' : 'local processes'} their units are recorded as not run.

` : '') - + hostSetup(rec); - } - const BOAT_SIZES = [['small', 'Small'], ['default', 'Default'], ['large', 'Large'], ['xlarge', 'XLarge']]; - function boatSizes(rec) { - const here = schema.standardHere || {}, why = here.boatSizes || {}, spec = here.boatSpecs || {}, size = rec.boat_size || 'small'; - const card = ([v, t]) => tile({attr: 'std-boat-size', key: v, on: size === v && !why[v], disabled: !!why[v], - note: why[v] ? 'Not on this account' : spec[v] || '', title: t}); - return '

VM size

' - + `
${BOAT_SIZES.map(card).join('')}
`; + + (outside.length ? `

${esc(outside.join(', '))} cannot run in a Boat VM: choose Hosts.

` : '') + + (dockerOnly.length && backend !== 'container' && agentRun ? `

${esc(dockerOnly.join(', '))}: ${dockerOnly.length > 1 ? 'these environments need' : 'this environment needs'} a container - ${sandboxes ? 'in a Boat VM' : 'on local processes'} their units are recorded as not run.

` : '') + + (sandboxes ? '' : hostSetup(rec)); } // What the chosen options of the benchmark need: an embedding model, requirements of the machine (standard.json host). function optionNeeds(rec) { @@ -564,6 +590,7 @@ }) .map(([n, o]) => { const cur = (rec.bench_options || {})[n] ?? o.default; + if (scope === 'where' && o.values.length === 1 && Object.values(o.needs || {}).some(x => x && x.vm)) return ''; // on the How it runs line if (scope === 'where' && o.values.length === 1) return `

${esc(o.label)}

${esc((o.names || {})[cur] || cur)}

`; if (o.pattern) return ``; const tileOf = v => { const need = (o.needs || {})[v] || {}; @@ -726,7 +753,7 @@ const ds = sets.length === 1 ? sets[0] : null, all = ds && ds.units ? String(ds.units) : 'all'; return '

Scope

' + field('Number of ' + u.nouns, 'limit', rec.limit, {type: 'number', min: 1, placeholder: all, hint: 'The first ones' + per + ', in its order.'}) - + field('Concurrent ' + u.nouns, 'concurrency', rec.concurrency, {type: 'number', min: 1, max: 32, placeholder: '1'}) + '
' + + field('Concurrent ' + u.nouns, 'concurrency', rec.concurrency, {type: 'number', min: 1, max: 32, placeholder: rec.backend === 'boat' ? String((schema.standardHere || {}).boatSlots || 'Boat\'s limit') : '1'}) + '' + '
' + field('Attempts per ' + u.noun, 'repeat', rec.repeat || '', {type: 'number', min: 1, max: 32, placeholder: '1'}) + field('Retries per attempt', 'retries', rec.retries ?? '', {type: 'number', min: 0, max: 9, placeholder: '0'}) + '
' // a conversation's most agent turns (recipe.py: run --max-turns): only a benchmark that holds conversations has turns @@ -969,7 +996,7 @@ root.addEventListener('submit',event=>{if(event.target.id==='nb-save-form'){event.preventDefault();save();}}); // Running from here: the runner script is posted with the target; the page follows the log until the run ends. const runPanel=()=>{ - const rec=r(), where=C.whereList(rec).map(w=>w==='local'?'this machine':C.machineLabel(w)).join(' + '); + const rec=r(), where=C.isStandard(rec)&&rec.backend==='boat'?'Boat sandboxes':C.whereList(rec).map(w=>w==='local'?'this machine':C.machineLabel(w)).join(' + '); if(!schema.local)return note('Run from the application host','Runs start from a browser on the machine Nullius runs on.'); // A provider not set in Settings: Run waits and says why. const blocked=C.providerBlocker(rec); @@ -1071,6 +1098,8 @@ else if(d.stdOther){const rec=r(),c=C.chosenOf(rec);rec.others=c.includes(d.stdOther)?c.filter(k=>k!==d.stdOther):[...c,d.stdOther];delete rec.other;if(rec.options)delete rec.options[d.stdOther];} // the standard's machines: each a toggle, one at least; several split the run into shards else if(d.stdWhere){const rec=r(),list=C.whereList(rec),on=list.includes(d.stdWhere),next=on?list.filter(w=>w!==d.stdWhere):[...list,d.stdWhere];if(!next.length)return;rec.where=next.length>1?next:next[0];} + // hosts or Boat sandboxes: a sandbox run starts from this machine alone + else if(d.stdMachines){const rec=r();if(d.stdMachines==='boat'){rec.backend='boat';rec.where='local';}else if(rec.backend==='boat')rec.backend='local';} else if(d.stdBackend){r().backend=d.stdBackend;} else if(d.stdBoatSize){r().boat_size=d.stdBoatSize;} else if(d.harborSource){r().source=d.harborSource;} diff --git a/app/tests/catalogue_docs.py b/app/tests/catalogue_docs.py index c087d490..7bef9284 100644 --- a/app/tests/catalogue_docs.py +++ b/app/tests/catalogue_docs.py @@ -231,7 +231,8 @@ def _values(spec: dict) -> str: def _needs_rows(name: str, spec: dict) -> list: - return [[f"`{name}`", f"`{val}`", "; ".join(f"{k}: {', '.join(w) if isinstance(w, list) else w}" for k, w in need.items())] + said = lambda k, w: f"a {w['memory_gb']:g} GB VM per unit" if k == "vm" else ", ".join(w) if isinstance(w, list) else w + return [[f"`{name}`", f"`{val}`", "; ".join(f"{k}: {said(k, w)}" for k, w in need.items())] for val, need in (spec.get("needs") or {}).items()] diff --git a/app/tests/test_standard_bench_options.py b/app/tests/test_standard_bench_options.py index 94e4dfa4..c66b46c7 100644 --- a/app/tests/test_standard_bench_options.py +++ b/app/tests/test_standard_bench_options.py @@ -38,6 +38,14 @@ def test_a_value_may_only_need_a_declared_requirement_and_a_known_dataset(self): problems = " ".join(definitions.problems({k: v for k, v in bad.items() if not k.startswith("_")}, "benchmark")) for said in ("options.retrieval.datasets", "needs.alltools.host", "needs.alltools.embeddings"): self.assertIn(said, problems) + # a unit's own VM is a Where option's, and says its memory in GB alone + for o, vm in ((TAU["options"]["retrieval"], {"memory_gb": 4}), (dict(TAU["options"]["retrieval"], step="where"), {"memory_gb": "4G"}), + (dict(TAU["options"]["retrieval"], step="where"), {"memory_gb": 4, "cpus": 4})): + bad = tau(options={"retrieval": dict(o, needs={"bm25": {"vm": vm}})}) + self.assertIn("needs.bm25.vm", " ".join(definitions.problems({k: v for k, v in bad.items() if not k.startswith("_")}, "benchmark"))) + good = tau(options={"retrieval": dict(TAU["options"]["retrieval"], step="where", needs={"bm25": {"vm": {"memory_gb": 4}}})}) + self.assertNotIn("needs.bm25", " ".join(definitions.problems({k: v for k, v in good.items() if not k.startswith("_")}, "benchmark"))) + self.assertEqual((definitions.vm_memory_gb(good, {"retrieval": "bm25"}), definitions.vm_memory_gb(good, {"retrieval": "terminal_use"})), (4, 0)) worse = tau(host={"sandbox": {"summary": "x", "check": ["srt; rm"], "install": {"brew": ["x"]}}}) problems = " ".join(definitions.problems({k: v for k, v in worse.items() if not k.startswith("_")}, "benchmark")) self.assertIn("host.sandbox.check", problems) diff --git a/app/tests/test_standard_boat.py b/app/tests/test_standard_boat.py index 54fcba34..972ccf62 100644 --- a/app/tests/test_standard_boat.py +++ b/app/tests/test_standard_boat.py @@ -119,6 +119,22 @@ def test_a_run_longer_than_the_accounts_starts_is_said(self): def test_slots_is_the_accounts_limit_of_vms_at_once(self): self.assertEqual(boat.slots(boat.Client("k", opener=FakeAPI([]))), 2) + def test_one_answer_of_its_limits_says_both_its_sizes_and_its_vms_at_once(self): + api = FakeAPI([]) + client = boat.Client("k", opener=api) + limits = client.limits() + self.assertEqual((boat.slots(client, limits), boat.machine_types(client, limits)["large"]), (2, "not during the free trial")) + self.assertEqual(len(api.calls), 1) + self.assertIsNone(boat.slots(client, {"maxActiveSandboxes": True})) + + def test_a_size_fits_a_units_own_vm_only_with_more_memory_than_it(self): + # measured: OSWorld's 4 GB guest is refused on small (4 GB, 2 free) and boots on default (8 GB) + self.assertFalse(boat.fits("small", 4)) + self.assertTrue(boat.fits("default", 4)) + self.assertTrue(boat.fits("small", 0)) + self.assertTrue(boat.fits("large", 4)) # its memory unmeasured: not refused here + self.assertEqual((boat.spec("small"), boat.spec("large")), ("2 vCPU, 4 GB", "")) + def test_sweep_deletes_only_this_jobs_vms(self): api = FakeAPI([]) self.assertEqual(boat.sweep("job.1", boat.Client("k", opener=api)), 1) @@ -514,11 +530,12 @@ def test_the_runs_size_is_every_vms_and_one_the_account_cannot_start_is_refused( with self.assertRaisesRegex(run.Refused, "Boat cannot start a large VM: not during the free trial"): run.Run(b, a, model="m", provider="custom", env=self.env, jobs_root=self.tmp, backend="boat", boat_size="large") - def test_a_check_runs_here_and_no_key_means_no_run(self): + def test_a_check_is_refused_not_run_here_and_no_key_means_no_run(self): b = definitions.get("benchmark", "fakebench", ROOT) b["controls"] = {"nop": {"source": "nullius", "from": "nothing said"}} nop = definitions.control(b, "nop") - self.assertEqual(run.Run(b, nop, model="m", provider="custom", env=self.env, jobs_root=self.tmp, backend="boat").backend, "local") + with self.assertRaisesRegex(run.Refused, "runs on a machine, not in a Boat VM"): + run.Run(b, nop, model="m", provider="custom", env=self.env, jobs_root=self.tmp, backend="boat") a = definitions.get("agent", "fake", ROOT) with mock.patch.object(boat, "unusable", lambda env=None, which=None: "no Boat API key: set it in Settings → Sandbox"): with self.assertRaisesRegex(run.Refused, "Boat backend cannot run here: no Boat API key"): @@ -539,16 +556,22 @@ def test_boat_needs_no_yes_and_runs_here_or_on_a_host(self): self.assertFalse(out["docker"]) sized = recipe.compile(a_recipe(backend="boat", boat_size="default"), defs(), boat_unusable=lambda here=True: None) self.assertIn("--boat-size default", sized["script"]) - self.assertIn("default VMs", sized["review"][-1][2]) + self.assertIn("Boat sandboxes · one Default VM per ", sized["review"][-1][2]) self.assertEqual([e["step"] for e in recipe.compile(a_recipe(backend="boat", boat_size="huge"), defs())["errors"]], [recipe.STEP["where"]]) - self.assertIn("boat backend", out["review"][-1][2]) - # on a host: the key is this machine's, the programs the host's (its prerequisites) + # as many units at once as the account runs, unless the recipe says fewer; the run caps it (boat.slots) + every = recipe.compile(a_recipe(backend="boat", concurrency=""), defs(), boat_unusable=lambda here=True: None) + self.assertIn(f"-j {recipe.MAX_CONCURRENT} ", every["script"]) + self.assertIn("as many at a time as Boat allows", next(r[2] for r in every["review"] if r[1] == "Scope")) + fewer = recipe.compile(a_recipe(backend="boat", concurrency=3), defs(), boat_unusable=lambda here=True: None) + self.assertIn("-j 3 ", fewer["script"]) + + def test_hosts_and_sandboxes_do_not_mix(self): asked = [] - out = recipe.compile(a_recipe(backend="boat", where="me@bench"), defs(), boat_unusable=lambda here=True: asked.append(here)) - self.assertEqual(out["errors"], []) - self.assertEqual(asked, [False]) - self.assertIn("--backend boat", out["script"]) + for where in ("me@bench", ["local", "me@bench"]): + out = recipe.compile(a_recipe(backend="boat", where=where), defs(), boat_unusable=lambda here=True: asked.append(here)) + self.assertEqual([e["message"] for e in out["errors"]], ["Hosts and Boat sandboxes do not mix: choose Hosts, or Sandboxes alone."]) + self.assertEqual(asked, []) def test_a_size_the_account_cannot_start_is_the_where_steps_error(self): problem = lambda size: "not during the free trial" if size == "large" else None @@ -558,6 +581,23 @@ def test_a_size_the_account_cannot_start_is_the_where_steps_error(self): out = recipe.compile(a_recipe(backend="boat", boat_size="default"), defs(), boat_unusable=lambda here=True: None, boat_size_problem=problem) self.assertEqual(out["errors"], []) + def test_a_benchmark_booting_its_own_vms_wants_a_size_that_holds_them_and_never_a_container(self): + # OSWorld's Original setup, on the bundled catalogue: the size is judged from its definition's VM, not its name + base = {"harness": recipe.HARNESS, "name": "os", "benchmark": "osworld2", "datasets": ["offline36"], "others": ["original"], + "model": "MiniMax-M3", "provider": "openrouter", "where": "local"} + errors = lambda **k: [e["message"] for e in recipe.compile(dict(base, **k), boat_unusable=lambda here=True: None)["errors"] + if e["step"] == recipe.STEP["where"]] + small = errors(backend="boat", boat_size="small") + self.assertIn("A Small VM is too small for OSWorld2, whose units boot a 4 GB VM: choose a larger size.", small) + # a setup or a check runs on a machine: on Sandboxes it is refused, never run here instead + self.assertIn("Original cannot run in a Boat VM: choose Hosts.", small) + self.assertIn("Original, Null control cannot run in a Boat VM: choose Hosts.", errors(backend="boat", boat_size="default", others=["original", "nop"])) + self.assertFalse([m for m in errors(backend="boat", boat_size="default") if "too small" in m]) + self.assertIn("OSWorld2 starts each unit's VM on the machine: choose local processes.", errors(backend="container")) + self.assertFalse([m for m in errors(backend="local") if "VM" in m]) + self.assertTrue(definitions.boots_vms(definitions.get("benchmark", "osworld1"))) + self.assertFalse(definitions.boots_vms(definitions.get("benchmark", "gsm8k"))) + def test_without_its_key_the_where_step_says_so(self): out = recipe.compile(a_recipe(backend="boat"), defs(), boat_unusable=lambda here=True: "no Boat API key: set it in Settings → Sandbox") self.assertEqual([e["step"] for e in out["errors"]], [recipe.STEP["where"]]) diff --git a/app/ui_tests/test_agents.py b/app/ui_tests/test_agents.py index e3a21583..7dd77445 100644 --- a/app/ui_tests/test_agents.py +++ b/app/ui_tests/test_agents.py @@ -523,9 +523,23 @@ def test_the_builder_is_given_the_controls_their_proofs_and_whether_docker_runs_ self.assertEqual([(c["key"], c["source"], c["proven"], c["failed"]) for c in ds["controls"]], [("nop", "nullius", None, None), ("oracle", "adapter", None, None)]) # no benchmark here needs a machine setup + sizes = [{"key": "small", "label": "Small", "spec": "2 vCPU, 4 GB", "memory_gb": 4, "why": None}, + {"key": "default", "label": "Default", "spec": "4 vCPU, 8 GB", "memory_gb": 8, "why": None}, + {"key": "large", "label": "Large", "spec": "", "memory_gb": None, "why": None}, + {"key": "xlarge", "label": "XLarge", "spec": "", "memory_gb": None, "why": None}] self.assertEqual(schema["standardHere"], {"docker": None, "boat": "no Boat API key: set it in Settings → Sandbox", - "boatKey": "no Boat API key: set it in Settings → Sandbox", "boatSizes": {}, "boatSpecs": {"small": "2 vCPU, 4 GB", "default": "4 vCPU, 8 GB"}, - "licences": {}, "host": {}}) + "boatSizes": sizes, "boatSlots": None, "licences": {}, "host": {}}) + + def test_the_builder_offers_the_boat_sizes_and_vms_at_once_its_account_says(self): + from nullius.standard import boat + self.fixture() + account = {"types": {"small": None, "default": None, "large": "not during the free trial"}, "slots": 2} + with patch.object(boat, "unusable", lambda here=True: None), patch.object(web.execute, "boat_account", lambda: account): + html = self.get("/new-run").get_data(as_text=True) + here = json.loads(re.search(r'', html, re.S).group(1))["standardHere"] + self.assertEqual(here["boatSlots"], 2) + self.assertEqual([(z["key"], z["why"]) for z in here["boatSizes"]], + [("small", None), ("default", None), ("large", "not during the free trial")]) # xlarge: not the account's def test_docker_is_refused_on_the_where_step_with_this_machines_reason(self): self.fixture() diff --git a/app/ui_tests/test_builder_where.py b/app/ui_tests/test_builder_where.py index 28dac581..168e1611 100644 --- a/app/ui_tests/test_builder_where.py +++ b/app/ui_tests/test_builder_where.py @@ -8,42 +8,91 @@ class StandardWhere(CompilerCase): - def draw(self, docker, edit=""): + SIZES = [{"key": "small", "label": "Small", "spec": "2 vCPU, 4 GB", "memory_gb": 4, "why": None}, + {"key": "default", "label": "Default", "spec": "4 vCPU, 8 GB", "memory_gb": 8, "why": None}, + {"key": "large", "label": "Large", "spec": "", "memory_gb": None, "why": "not during the free trial"}] + + def draw(self, docker, edit="", backends=("local", "docker"), boat=None, slots=2): """standardWhere and what it calls, cut from the page script, run against the core: {benchmark backend: html}; - ``edit``, JavaScript changing the recipe ``r`` first.""" + ``edit``, JavaScript changing the recipe ``r`` first; ``boat``, why Boat cannot run here.""" page = (APP / "static/builder/command-builder.js").read_text(encoding="utf-8") cut = lambda start, end: page[page.index(start):page.index(end, page.index(start))] line = lambda name: re.search(r"^ const %s = .*$" % name, page, re.M).group(0) + "\n" - schema = dict(self.schema, hosts=[], standardHere={"docker": docker, "licences": {}, "host": {}}) + schema = dict(self.schema, hosts=[], standardHere={"docker": docker, "licences": {}, "host": {}, "boat": boat, + "boatSizes": self.SIZES, "boatSlots": slots}) program = ("const schema=" + json.dumps(schema) + ";const C=require(" + json.dumps(str(APP / "static/builder/command-builder-core.js")) + ")(schema);\n" - + line("esc") + line("button") + cut(" const tile = t =>", " const picker = ") + + line("esc") + line("button") + cut(" const unitsOf = ", " // the first clause") + line("words") + + cut(" const tile = t =>", " const picker = ") + cut(" const consentBox = ", " async function hostReq(") + cut(" function benchOptions(", " // Only credentials") - + "const out={};for(const b of C.standard().benchmarks)for(const backend of ['local','docker']){" + + "const out={};for(const b of C.standard().benchmarks)for(const backend of " + json.dumps(list(backends)) + "){" + "const r=C.createStandard(b.key);r.backend=backend;" + edit + "try{out[b.key+' '+backend]=standardWhere(r);}catch(e){out[b.key+' '+backend]=String(e);}}" + "process.stdout.write(JSON.stringify(out));") return json.loads(subprocess.run([self.node, "-"], input=program, check=True, capture_output=True, text=True).stdout) + def test_hosts_or_sandboxes_never_both_and_a_sandbox_is_its_own_isolation(self): + hosts, boxes = self.draw(None), self.draw(None, backends=("boat",)) + card = lambda html, v: re.search(r'`; const machine = (v, t, sub) => machineTile('std-where', v, t, sub, list.includes(v)); - const how = (v, t, sub, why) => tile({attr: 'std-backend', key: v, on: backend === v && !why, disabled: !!why, why, note: why ? 'Not usable here' : '', mark: v === 'local' ? icon('terminal') : icon('container'), title: t, tip: sub}); + const how = (v, t, sub, why) => tile({attr: 'std-backend', key: v, on: backend === v && !why, disabled: !!why, why, note: why ? 'Not usable here' : v === 'local' ? 'Not sandboxed' : `Each ${noun} in a container`, mark: v === 'local' ? icon('terminal') : icon('container'), title: t, tip: sub}); const dockerOnly = Object.entries((b || {}).environments || {}).filter(([, e]) => !e.local_ok).map(([n]) => n); const web = agentRun && Object.values((b || {}).environments || {}).some(e => e.network); // an agent's public web proxy // a VM benchmark's provider of one value goes on the fixed line; several are its cards (benchOptions) @@ -502,15 +500,15 @@ disabled: !!z.why, why: z.why, note: z.note || z.spec, title: z.label})).join('')}`; } return '

Where does it run?

' - + `
${kind('hosts', 'Hosts', icon('machine'), H.length ? `This machine + ${H.length} ${H.length > 1 ? 'hosts' : 'host'}` : 'This machine', 'One shard per machine.', '')}` - + kind('boat', 'Sandboxes', BOAT_MARK, here.boatSlots ? `Boat · ${here.boatSlots} at once` : 'Boat', `One Boat VM per ${noun}.`, here.boat || '') + '
' - + (sandboxes ? sizes - : '

Machines

' - + `
${machine('local', 'This machine', 'Nullius runs it here, in runs/' + rec.name + '.')}${H.map(h => machine(h.entry, h.label, 'Over ssh: the standard\'s code at this commit is sent there, uv put there if missing; its job comes back as it runs.')).join('')}
` - + (H.length ? (list.some(w => w !== 'local') ? button('Test connection', 'id="nb-test-host"') + ' ' : '') : `

Hosts are named in ${button('Settings → Hosts', 'data-href="/settings#HOSTS"', 'nb-inlinebutton')}.

`)) - + '

How it runs

' + + `
${kind('hosts', 'Hosts')}${kind('boat', 'Sandboxes', here.boat || '')}
` + + (sandboxes ? `
BoatOne VM per ${esc(noun)}${here.boatSlots ? `${here.boatSlots} at once on this account` : ''}${here.boat ? `${esc(here.boat)}` : ''}
` + sizes + : `
${machine('local', 'This machine', 'Runs here')}${H.map(h => machine(h.entry, h.label, 'Over SSH')).join('')}
` + + `

${list.length} ${list.length === 1 ? 'shard' : 'shards, merged into one job'}

` + + (list.some(w => w !== 'local') ? button('Test connection', 'id="nb-test-host"') + ' ' : '')) + + '

Isolation

' + (sandboxes || vms - ? `

Each ${esc(noun)} in its own VM${provider ? ` · ${esc(provider.label)}: ${esc((provider.names || {})[provider.values[0]] || words(provider.values[0]))}` : ''}

` + ? `
${vms ? `Each ${esc(noun)} in its own VM` : `The agent alone in each ${esc(noun)}'s VM`}` + + `${vms ? (provider ? (sandboxes ? `${esc((provider.names || {})[provider.values[0]] || words(provider.values[0]))}, inside the ${esc(noun)}'s Boat VM` : `${esc(provider.label)}: ${esc((provider.names || {})[provider.values[0]] || words(provider.values[0]))}`) : '') : 'Answers and keys stay on this machine'}
` : `
${how('local', 'Local processes', 'Its commands run as you, not sandboxed.', '')}` + how('container', 'Container', web ? 'Sandboxed: each unit in a container whose only ways out are its relay and a public web proxy.' : 'Sandboxed: each unit in a container whose only way out is its relay.', containerWhy) + '
') diff --git a/app/ui_tests/test_builder_where.py b/app/ui_tests/test_builder_where.py index 66f30cab..cddad841 100644 --- a/app/ui_tests/test_builder_where.py +++ b/app/ui_tests/test_builder_where.py @@ -38,18 +38,23 @@ def test_hosts_or_sandboxes_never_both_and_a_sandbox_is_its_own_isolation(self): self.assertTrue(html.startswith("Sandboxes", html) - self.assertIn("Boat · 2 at once", html) # the account's VMs at once, from its API + self.assertIn(">Sandboxes", html) + self.assertIn("2 at once on this account", html) # the account's VMs at once, from its API self.assertNotIn("data-std-where", html) # no host to add to a sandbox run self.assertNotIn("data-std-backend", html) # nothing to choose: the VM is the sandbox self.assertNotIn("not sandboxed", html.lower()) - self.assertRegex(html, r'

Each [a-z -]+ in its own VM') + self.assertIn('class="nb-fixed std-isolation"', html) self.assertIn('data-std-boat-size="large" aria-pressed="false" disabled aria-disabled="true" title="Not on this account: not during the free trial"', html) for name, html in hosts.items(): with self.subTest(name): self.assertIn('aria-pressed="true"', card(html, "hosts")) self.assertIn('data-std-where="local"', html) self.assertNotIn("data-std-boat-size", html) + ordinary = boxes["humaneval boat"] + self.assertIn("The agent alone in each problem's VM", ordinary) + self.assertIn("Answers and keys stay on this machine", ordinary) + self.assertIn('class="std-boat-card"', ordinary) + self.assertNotIn("shards, merged", ordinary) self.assertNotIn("at once", self.draw(None, backends=("boat",), slots=None)["gsm8k boat"]) # the API said nothing: no number gone = self.draw(None, backends=("boat",), boat="no Boat API key: set it in Settings → Sandbox")["gsm8k boat"] self.assertIn('disabled aria-disabled="true" title="no Boat API key: set it in Settings → Sandbox"', card(gone, "boat")) @@ -60,13 +65,13 @@ def test_a_benchmark_booting_its_own_vms_asks_no_isolation_and_refuses_a_size_to with self.subTest(key): hosts = self.draw(None)[key + " local"] self.assertNotIn("data-std-backend", hosts) - self.assertIn('

Each task in its own VM · VM provider: Docker

', hosts) + self.assertIn('Each task in its own VMVM provider: Docker', hosts) self.assertEqual(hosts.count("VM provider"), 1) boxes = self.draw(None, backends=("boat",))[key + " boat"] label = "OSWorld1" if key == "osworld1" else "OSWorld2" self.assertIn(f'data-std-boat-size="small" aria-pressed="false" disabled aria-disabled="true" title="Too small for {label}'s 4 GB VM"', boxes) self.assertIn('data-std-boat-size="default" aria-pressed="true"', boxes) # the smallest that fits, chosen for it - self.assertIn('

Each task in its own VM · VM provider: Docker

', boxes) + self.assertIn("Each task in its own VMDocker, inside the task's Boat VM", boxes) self.assertNotIn("cannot run in a Boat VM", boxes) # Null controls run whole in each VM, including OSWorld2's grader. nop = self.draw(None, "r.others=['nop'];", backends=("boat",)) From 0f2cbb26f07f0542c53e947229af4cf54ef64b43 Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Thu, 19 Nov 2026 13:59:13 +0100 Subject: [PATCH 5/6] fix(builder): compact machine requirements and retain command copy --- .../benchmarks/osworld2/standard.json | 4 +- app/static/builder/command-builder.css | 14 +++-- app/static/builder/command-builder.js | 52 ++++++++++--------- app/ui_tests/test_builder_where.py | 21 +++++--- app/ui_tests/test_catalogue_routes.py | 2 +- 5 files changed, 56 insertions(+), 37 deletions(-) diff --git a/app/nullius/catalogue/benchmarks/osworld2/standard.json b/app/nullius/catalogue/benchmarks/osworld2/standard.json index 5378d2a7..8b953451 100644 --- a/app/nullius/catalogue/benchmarks/osworld2/standard.json +++ b/app/nullius/catalogue/benchmarks/osworld2/standard.json @@ -249,8 +249,8 @@ ] }, "scoring_programs": { - "label": "Scoring programs", - "summary": "ImageMagick, ffmpeg and tesseract, used to score 6 tasks.", + "label": "Scoring tools", + "summary": "ImageMagick · FFmpeg · Tesseract", "check": [ "identify", "convert", diff --git a/app/static/builder/command-builder.css b/app/static/builder/command-builder.css index 9d8c4ffa..769c48da 100644 --- a/app/static/builder/command-builder.css +++ b/app/static/builder/command-builder.css @@ -188,9 +188,17 @@ #nullius-builder .nb-pick-none { color: var(--nb-muted); font-size: 13px; margin: 0; } @media (max-width: 640px) { #nullius-builder .nb-pick-chips + .nb-pick-chips { padding-left: 0; border-left: 0; } #nullius-builder .nb-pick-find { max-width: none; } #nullius-builder .nb-pick-find input { font-size: 16px; } } /* The Where step's machine setup: each chosen machine, its state for the search's requirements, check and install. */ -#nullius-builder .nb-hostreq { list-style: none; margin: 0 0 12px; padding: 0; display: grid; gap: 8px; } -#nullius-builder .nb-hostreq li { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; padding: 10px 12px; border: 1px solid var(--nb-line); border-radius: 8px; font-size: 12.5px; } -#nullius-builder .nb-hostreq small { color: var(--nb-muted); } +#nullius-builder .std-hostreq { padding: 16px; border: 1px solid var(--nb-line); border-radius: 8px; font-size: 13px; } +#nullius-builder .std-hostreq-row { display: flex; flex-wrap: wrap; align-items: center; gap: 12px; } +#nullius-builder .std-hostreq-row + .std-hostreq-row { margin-top: 12px; } +#nullius-builder .std-hostreq-names { flex: 1 1 240px; } +#nullius-builder .std-hostreq-actions { display: flex; flex-wrap: wrap; align-items: center; gap: 12px; } +#nullius-builder .std-hostreq small { color: var(--nb-muted); } +#nullius-builder .std-hostreq details { margin-top: 12px; } +#nullius-builder .std-hostreq summary { color: var(--nb-muted); cursor: pointer; } +#nullius-builder .std-hostreq .std-code { margin: 12px 0 0; display: flex; align-items: center; } +#nullius-builder .std-hostreq .std-code-head { order: 1; border: 0; } +#nullius-builder .std-hostreq .std-code pre { flex: 1; min-width: 0; overflow-wrap: anywhere; } #nullius-builder .std-code { border: 1px solid var(--nb-line); border-radius: 8px; background: var(--nb-bg); overflow: hidden; margin-bottom: 12px; } #nullius-builder .std-code-head { display: flex; justify-content: space-between; align-items: center; gap: 8px; padding: 5px 10px; border-bottom: 1px solid var(--nb-line); font: 11.5px var(--mono); color: var(--nb-muted); } #nullius-builder .std-copy { border: 1px solid var(--nb-line); background: var(--nb-surface); color: var(--nb-muted); border-radius: 6px; padding: 2px 9px; font-size: 11.5px; cursor: pointer; } diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index 41750ff9..83831868 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -504,7 +504,6 @@ + `
${kind('hosts', 'Hosts')}${kind('boat', 'Sandboxes', here.boat || '')}
` + (sandboxes ? `
BoatOne VM per ${esc(noun)}${here.boatSlots ? `${here.boatSlots} at once on this account` : ''}${here.boat ? `${esc(here.boat)}` : ''}
` + sizes : `
${machine('local', 'This machine', 'Runs here')}${H.map(h => machine(h.entry, h.label, 'Over SSH')).join('')}
` - + `

${list.length} ${list.length === 1 ? 'shard' : 'shards, merged into one job'}

` + (list.some(w => w !== 'local') ? button('Test connection', 'id="nb-test-host"') + ' ' : '')) + '

Isolation

' + (sandboxes || vms @@ -533,7 +532,7 @@ } // The machines' setup for those requirements: each chosen machine's state, a check, an install on an explicit // click (sudo -n: a machine whose sudo asks for a password is told the commands instead), the commands to copy. - const hostState = {}; // entry -> {missing, error, said} + const hostState = {}; // benchmark:entry:requirement -> {missing, error, said} function hostSetup(rec) { const b = C.standardBench(rec), need = optionNeeds(rec).host; if (!b || !need.length) return ''; @@ -542,31 +541,36 @@ if ((rec.backend || 'local') === 'container' && chosen.length && chosen.every(o => o.setup)) return ''; const reqs = need.map(n => [n, (b.host || {})[n]]).filter(([, r]) => r); const local = ((schema.standardHere || {}).host || {})[b.key] || {}; - // what installs them; none for a machine's own hardware (a GPU), which no package puts there - const lines = reqs.flatMap(([n]) => b.host[n].install).filter((x, i, a) => a.indexOf(x) === i); - const line = w => { - const st = hostState[w] || (w === 'local' ? {missing: Object.fromEntries(need.filter(n => local[n]).map(n => [n, local[n]]))} : null); - const gone = st && st.missing ? Object.entries(st.missing).filter(([n]) => need.includes(n)) : []; - const state = !st ? 'not checked' : st.error ? `${esc(st.error)}` - : gone.length ? `${esc(gone.map(([, ps]) => ps.join(', ')).join(', '))}` : 'ready ✓'; - return `
  • ${esc(w === 'local' ? 'This machine' : C.machineLabel(w))} ${state} ${button('Check', `data-hostreq-check="${esc(w)}"`, 'nb-inlinebutton')}${lines.length && (gone.length || (st && st.error)) ? ' ' + button('Install for me', `data-hostreq-install="${esc(w)}"`, 'nb-inlinebutton') : ''}${st && st.said ? ` ${esc(st.said)}` : ''}
  • `; - }; - return `

    Machine setup

    Needed by the options you chose, on each machine it runs on
    ` - + reqs.map(([n, r]) => `

    ${esc(r.label)}: ${esc(r.summary)}

    `).join('') - + `
      ${C.whereList(rec).map(line).join('')}
    ` - + (lines.length ? `
    or run it yourself, on each machine
    ${esc(lines.join('\n'))}
    ` : ''); + return reqs.map(([name, req]) => { + const lines = req.install || []; + const machines = C.whereList(rec); + const rows = machines.map(w => { + const st = hostState[b.key + ':' + w + ':' + name] || (w === 'local' ? {missing: local[name] ? {[name]: local[name]} : {}} : null); + const missing = st && st.missing && st.missing[name]; + const state = !st ? 'Not checked' : st.error ? st.error : missing && missing.length ? 'Missing' : 'Ready'; + const attrs = `data-hostreq-name="${esc(name)}"`; + return `
    ${machines.length > 1 || w !== 'local' ? `${esc(w === 'local' ? 'This machine' : C.machineLabel(w))}` : ''}` + + `${esc(req.summary || req.label)}` + + `
    ${esc(state)}` + + button('Check', `data-hostreq-check="${esc(w)}" ${attrs}`, 'nb-button') + + (lines.length && (missing && missing.length || st && st.error) ? button('Install', `data-hostreq-install="${esc(w)}" ${attrs}`, 'nb-button nb-primary') : '') + + `
    ${st && st.said ? `${esc(st.said)}` : ''}
    `; + }).join(''); + return `

    ${esc(req.label)}

    ${rows}` + + (lines.length ? `
    Install manually
    ${esc(lines.join('\n'))}
    ` : '') + '
    '; + }).join(''); } - async function hostReq(entry, what) { - const rec = r(), b = C.standardBench(rec); - hostState[entry] = {...(hostState[entry] || {}), said: what === 'install' ? 'installing…' : 'checking…', error: ''}; + async function hostReq(entry, what, name) { + const rec = r(), b = C.standardBench(rec), key = b.key + ':' + entry + ':' + name; + hostState[key] = {...(hostState[key] || {}), said: what === 'install' ? 'Installing…' : 'Checking…', error: ''}; renderForm(); try { const res = await nulliusFetch('/new-run/host/' + what, {method: 'POST', headers: {'X-CSRF-Token': schema.csrf}, - body: new URLSearchParams({entry, benchmark: b.key, requirements: optionNeeds(rec).host.join(',')})}); + body: new URLSearchParams({entry, benchmark: b.key, requirements: name})}); const j = await res.json(); - if (what === 'install' && j.ok) return hostReq(entry, 'check'); - hostState[entry] = j.ok ? {missing: j.missing || {}} : {...(hostState[entry] || {}), error: j.error || ('HTTP ' + res.status), said: j.manual ? 'run the commands below by hand' : ''}; - } catch (e) { hostState[entry] = {error: String(e)}; } + if (what === 'install' && j.ok) return hostReq(entry, 'check', name); + hostState[key] = j.ok ? {missing: j.missing || {}} : {...(hostState[key] || {}), error: j.error || ('HTTP ' + res.status), said: j.manual ? 'Use Install manually below' : ''}; + } catch (e) { hostState[key] = {error: String(e)}; } renderForm(); } // A benchmark's own options (a search, a mode), for the chosen datasets they apply to, and what a value needs of this @@ -1077,8 +1081,8 @@ // key, the keyboard goes back to that field if(d.servedRemove!==undefined){event.preventDefault();C.removeServed(r(),d.servedRole,d.servedRemove);syncServed();refreshPlan();if(event.detail===0)root.querySelector(`input[data-served-by="${d.servedRole}"]`)?.focus();return;} if(d.benchOpt!==undefined){const rec=r(),o=((C.standardBench(rec)||{}).options||{})[d.benchOpt]||{};rec.bench_options={...(rec.bench_options||{})};if(d.benchVal===o.default)delete rec.bench_options[d.benchOpt];else rec.bench_options[d.benchOpt]=d.benchVal;renderForm();refreshPlan();return;} - if(d.hostreqCheck!==undefined){hostReq(d.hostreqCheck,'check');return;} - if(d.hostreqInstall!==undefined){hostReq(d.hostreqInstall,'install');return;} + if(d.hostreqCheck!==undefined){hostReq(d.hostreqCheck,'check',d.hostreqName);return;} + if(d.hostreqInstall!==undefined){hostReq(d.hostreqInstall,'install',d.hostreqName);return;} if(d.copyCode!==undefined){const t=b.closest('.std-code').querySelector('code').textContent;try{await navigator.clipboard.writeText(t);b.textContent='Copied';}catch{b.textContent='Select it';}setTimeout(()=>{b.textContent='Copy';},1500);return;} if(d.pickCat!==undefined){pickOf().cat=d.pickCat;applyPick();return;} if(d.pickFam!==undefined){pickOf().fam=d.pickFam;applyPick();return;} diff --git a/app/ui_tests/test_builder_where.py b/app/ui_tests/test_builder_where.py index cddad841..478b4fb5 100644 --- a/app/ui_tests/test_builder_where.py +++ b/app/ui_tests/test_builder_where.py @@ -12,13 +12,13 @@ class StandardWhere(CompilerCase): {"key": "default", "label": "Default", "spec": "4 vCPU, 8 GB", "memory_gb": 8, "why": None}, {"key": "large", "label": "Large", "spec": "", "memory_gb": None, "why": "not during the free trial"}] - def draw(self, docker, edit="", backends=("local", "docker"), boat=None, slots=2): + def draw(self, docker, edit="", backends=("local", "docker"), boat=None, slots=2, host=None): """standardWhere and what it calls, cut from the page script, run against the core: {benchmark backend: html}; ``edit``, JavaScript changing the recipe ``r`` first; ``boat``, why Boat cannot run here.""" page = (APP / "static/builder/command-builder.js").read_text(encoding="utf-8") cut = lambda start, end: page[page.index(start):page.index(end, page.index(start))] line = lambda name: re.search(r"^ const %s = .*$" % name, page, re.M).group(0) + "\n" - schema = dict(self.schema, hosts=[], standardHere={"docker": docker, "licences": {}, "host": {}, "boat": boat, + schema = dict(self.schema, hosts=[], standardHere={"docker": docker, "licences": {}, "host": host or {}, "boat": boat, "boatSizes": self.SIZES, "boatSlots": slots}) program = ("const schema=" + json.dumps(schema) + ";const C=require(" + json.dumps(str(APP / "static/builder/command-builder-core.js")) + ")(schema);\n" + line("esc") + line("button") + cut(" const unitsOf = ", " // the first clause") + line("words") @@ -117,11 +117,18 @@ def test_docker_names_the_public_web_proxy_only_for_an_agent_whose_environment_d self.assertEqual("public web proxy" in tip(html), proxy) self.assertNotIn("public web proxy", tip(setup[name])) # an Original setup gets no proxy - def test_a_machine_setup_is_said_to_be_needed_by_the_options_chosen(self): - html = self.draw(None)["osworld1 local"] - self.assertIn("

    Machine setup

    Needed by the options you chose, on each machine it runs on", html) - self.assertIn("docker: Docker on the machine the driver runs on", html) - self.assertIn("Docker: Docker without sudo", self.draw(None)["osworld2 local"]) + def test_machine_requirements_are_compact_and_manual_commands_keep_copy(self): + html = self.draw(None, host={"osworld2": {"scoring_programs": ["identify not on PATH"]}})["osworld2 local"] + self.assertIn('data-hostreq-install="local" data-hostreq-name="scoring_programs"', html) + self.assertIn(">Missing", html) + self.assertNotIn("Needed by the options", html) + self.assertNotIn("1 shard", html) + self.assertIn("

    Scoring tools

    ", html) + self.assertIn("ImageMagick · FFmpeg · Tesseract", html) + self.assertIn('
    Install manually', html) + self.assertIn('data-copy-code>Copy', html) + self.assertIn('data-hostreq-name="scoring_programs"', html) + self.assertIn('sudo apt-get install', html) def test_an_option_shows_its_example_and_a_setup_its_label(self): page = (APP / "static/builder/command-builder.js").read_text(encoding="utf-8") diff --git a/app/ui_tests/test_catalogue_routes.py b/app/ui_tests/test_catalogue_routes.py index 9b3de520..7e8430cf 100644 --- a/app/ui_tests/test_catalogue_routes.py +++ b/app/ui_tests/test_catalogue_routes.py @@ -249,7 +249,7 @@ def test_a_machine_setup_says_what_checks_it_and_what_installs_it(self): def test_a_machine_setup_shows_its_label_and_its_key_without_one(self): osworld2 = self.page("osworld2") - for label in ("Docker", "Scoring programs"): + for label in ("Docker", "Scoring tools"): self.assertIn("Setup: " + label, osworld2) self.assertIn("needs " + label, osworld2) self.assertNotIn("Setup: scoring_programs", osworld2) From d5869897b4029d6bb9b0c9ff2244463411e0f45c Mon Sep 17 00:00:00 2001 From: Etienne Filhol Date: Thu, 19 Nov 2026 14:22:44 +0100 Subject: [PATCH 6/6] docs(builder): trim Boat guidance to choices and limits --- docs/builder.md | 2 +- docs/cli/run.md | 2 -- docs/standard.md | 10 ++++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/builder.md b/docs/builder.md index 34b84985..0a1d1928 100644 --- a/docs/builder.md +++ b/docs/builder.md @@ -28,7 +28,7 @@ Then [read its results](results.md). | Agent (*Memory* on a memory benchmark) | Reference agents first, or **No agent · Original** for a direct-model benchmark. Memory benchmarks offer **Original memory** alongside compatible memory capabilities. Controls follow separately; each choice has its own settings tab | | Model | **Like the paper** keeps what the paper fixed: the model picker shows only documented paper models when the benchmark lists them (the filter applies to the selected agent or capability too), and the user simulator and judge are their official models. A benchmark that lists no paper models but has an official simulator or judge leaves the model to you. **Like the leaderboard**, on a benchmark that declares its leaderboard's rules, leaves the model to you and fixes what the leaderboard does: its user simulator and judge, the options it sets and at least its attempts per unit. **Any of your choice** opens the provider's full list and lets the simulator and judge be changed. Nothing fixed by the paper: the paper choice is disabled. **Served by**, a row after each model field's when the provider is OpenRouter, waits for the provider list and stays hidden when it has exactly one entry and the field has no pin; an existing pin always shows, as does an empty or failed list. Two endpoints of one provider count as two entries. It chooses the serving providers that may answer that model, tried in their order and no other ([Run a benchmark](cli/run.md#serving-providers)). Its picker is the model's: typing filters the model's providers, and each pick, or a slug typed and Enter, joins the list under it, each removable. Empty: any of them. Review shows them in its Model row | | Limits | Task limit (empty is every task), a task-name glob, concurrency, attempts per unit, retries per attempt, timeouts in minutes. On the Nullius runner, a [spending cap](cli/run.md#a-spending-cap) for the run and one per unit, turns per conversation for a benchmark that holds conversations, a [cap on model calls](cli/run.md#a-cap-on-model-calls) per unit, [time limits](cli/run.md#time-limits) per phase in minutes, forced on every participant (empty: the benchmark's or agent's own value in text colour, Nullius's default in grey), then the benchmark's own bounds in a last section of its name (τ³-bench's steps and tool errors per conversation, OSWorld's steps per task); a [list of units](cli/run.md#a-list-of-units) is set only on the command line | -| Where | *Hosts* or *Sandboxes*, never both. Hosts: this machine, Settings hosts, one shard each, merged into one job; on them *local processes*, as you, with the explicit yes to running unsandboxed, or *Container*, on Docker or Podman. Sandboxes: a Boat VM per unit, as many at once as the account allows, in a size the account offers; a size too small for a unit's own VM is refused. A benchmark whose units boot their own VM (OSWorld) runs as local processes, its VM provider fixed when it has one value. A benchmark option's machine setup is installed on a click (`sudo -n`), or its commands are given | +| Where | **Hosts**: this machine or Settings hosts. **Sandboxes**: Boat VMs, within the account’s sizes and capacity. Choose isolation where available; VM benchmarks keep their own VM provider | | Review | One row per decision and the prerequisites. **Run** | Original setups run on local processes or in Docker; Docker changes isolation only. diff --git a/docs/cli/run.md b/docs/cli/run.md index 9079e9ea..29d49438 100644 --- a/docs/cli/run.md +++ b/docs/cli/run.md @@ -29,8 +29,6 @@ nullius standard run locomo model --model gpt-4o-mini | `--units FILE` | Only the units the file lists: [below](#a-list-of-units) | | `--unsandboxed` | An agent runs as you; else `--backend container` | | `--backend` | `local` (default); `container`, each unit in a container with its relay; or [`boat`](../standard.md#the-boat-backend), a VM per unit | -| `--shard-by` | `id` (default): a unit's shard by its id's hash; `position`: by its place in the dataset | -| `--relay-container` | Local task processes with a separate relay container on their host; used inside whole-task Boat VMs | | `--boat-size` | Boat's VM size: `small` (default), `default`, `large`, `xlarge` | | `--option NAME=VALUE` | An agent's or capability's option (repeatable) | | `--bench-option NAME=VALUE` | A benchmark's option (repeatable), read as `NULLIUS_OPTION_` | diff --git a/docs/standard.md b/docs/standard.md index ec9980cb..c888368b 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -181,18 +181,20 @@ A rootful Docker with `userns-remap` maps the uids of each container to sub-uids `--backend boat`: each unit's agent or capability in its own [boat.dev](https://boat.dev) VM. Driver, relay, spending cap and scoring stay with the run. -Refused before any unit: a computer-port run (its server's socket is on this machine), an Original setup or a check (they run on a machine). Choose local processes or the container backend. +Except for VM benchmarks below, computer-port runs, Original setups and checks require Hosts. -A benchmark whose units boot their own VM (OSWorld) runs each unit whole in its VM instead: driver, desktop VM and all, as a one-unit shard (`--shard-by position`) of this runner's snapshot, merged here into one job (`standard/boat_units.py`). Each VM installs what the unit needs. Model calls use a separate relay container inside each Boat VM; only the relay receives the real model key. Declared benchmark credentials also enter the VM. A run's spending cap is divided equally between its tasks; unused shares are not transferred. `--units` and `--shard` are unavailable on this path; choose a dataset and limit instead. +VM benchmarks such as OSWorld run each task entirely inside Boat, including scoring. Model keys and declared benchmark credentials enter those VMs. + +For these runs, spending caps are split equally across tasks; unused shares cannot transfer. Use a dataset and limit instead of `--units` or `--shard`. | Piece | Rule | | --- | --- | | Key | Settings → Sandbox, or `BOAT_API_KEY` | -| VM | One per unit, `--boat-size` (`small` by default). Always deleted. Free trial: 2 hours at most. Review refuses a size without more memory than a unit's own VM (`needs..vm`) | +| VM | One per unit, `--boat-size` (`small` by default). Always deleted. Free trial: 2 hours at most. VM benchmarks need more RAM than their guest VM | | Machines | Runs from this machine; never mixed with hosts | | Units at a time | The account's VM limit; a recipe may set fewer. The run warns when it cannot start every unit | | Agent time | Excludes the VM's start-up | -| Isolation | Agent-only VMs reach loopback only: no internet, no DNS, no model key. Whole-task VMs run the driver and desktop with a separate relay container | +| Isolation | Agent-only VMs reach loopback only: no internet, no DNS, no model key. VM benchmarks isolate the whole task | | Files | A link into your home or keys refuses the unit. Links out of the workspace are not copied back | | Code checks | A `runs_code` check needs Docker where the run runs | | Not run | Units needing the public web, their own image, a licence file or server of this machine, or MCP servers. No GPUs |