diff --git a/app/nullius/catalogue/benchmarks/osworld1/standard.json b/app/nullius/catalogue/benchmarks/osworld1/standard.json index 5eb63b92..a1d5a253 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/standard.json +++ b/app/nullius/catalogue/benchmarks/osworld1/standard.json @@ -254,7 +254,10 @@ "docker": { "host": [ "docker" - ] + ], + "vm": { + "memory_gb": 4 + } } } }, diff --git a/app/nullius/catalogue/benchmarks/osworld2/standard.json b/app/nullius/catalogue/benchmarks/osworld2/standard.json index 55930670..8b953451 100644 --- a/app/nullius/catalogue/benchmarks/osworld2/standard.json +++ b/app/nullius/catalogue/benchmarks/osworld2/standard.json @@ -192,7 +192,10 @@ "host": [ "docker", "scoring_programs" - ] + ], + "vm": { + "memory_gb": 4 + } } } }, @@ -246,8 +249,8 @@ ] }, "scoring_programs": { - "label": "Scoring programs", - "summary": "ImageMagick, ffmpeg and tesseract, used to score 6 tasks.", + "label": "Scoring tools", + "summary": "ImageMagick · FFmpeg · Tesseract", "check": [ "identify", "convert", diff --git a/app/nullius/standard/boat.py b/app/nullius/standard/boat.py index 1eb6fe8a..ce7cde06 100644 --- a/app/nullius/standard/boat.py +++ b/app/nullius/standard/boat.py @@ -56,8 +56,10 @@ API = "https://boat.dev/api/v1" STORE, NAME = "sandbox-provider-boat", "BOAT_API_KEY" -# the VM runs the agent's process, never its model: the smallest is enough -SIZES = {"small": "2 vCPU, 4 GB", "default": "4 vCPU, 8 GB", "large": "", "xlarge": ""} +# Boat's machine types, by name: its label and, where measured, its vCPUs and memory. The VM runs the agent's +# process, never its model: the smallest is enough, unless a task boots a VM of its own inside it (fits). +SIZES = {"small": {"label": "Small", "cpus": 2, "memory_gb": 4}, "default": {"label": "Default", "cpus": 4, "memory_gb": 8}, + "large": {"label": "Large"}, "xlarge": {"label": "XLarge"}} SIZE = "small" TTL = 2 * 3600 # a VM's auto-stop at least, and a free trial's at most USER = "agent" @@ -153,10 +155,11 @@ def said(d: dict) -> str: return str(d.get("message") or d.get("code") or d.get("status") or "no answer") -def slots(client: Client) -> Optional[int]: - """How many VMs the account may run at once; None when its API does not say.""" - n = client.limits().get("maxActiveSandboxes") - return n if isinstance(n, int) and n > 0 else None +def slots(client: Client, limits: Optional[dict] = None) -> Optional[int]: + """How many VMs the account may run at once; None when its API does not say. ``limits``: its + answer to ``GET /limits`` when already asked.""" + n = (client.limits() if limits is None else limits).get("maxActiveSandboxes") + return n if isinstance(n, int) and not isinstance(n, bool) and n > 0 else None def starts_problem(client: Client, n: int) -> Optional[str]: @@ -172,16 +175,30 @@ def starts_problem(client: Client, n: int) -> Optional[str]: return None -def machine_types(client: Client) -> Dict[str, Optional[str]]: +def machine_types(client: Client, limits: Optional[dict] = None) -> Dict[str, Optional[str]]: """Each machine type the account knows: None when it may start one, else boat's reason why not ("not during the free trial"). Empty when its API does not say.""" out = {} - for t in client.limits().get("machineTypes") or []: + for t in (client.limits() if limits is None else limits).get("machineTypes") or []: if isinstance(t, dict) and isinstance(t.get("type"), str): out[t["type"]] = None if t.get("available") else str(t.get("reason") or "not available on this account") return out +def spec(size: str) -> str: + """A size's vCPUs and memory as a card says them, or "" when unmeasured.""" + z = SIZES.get(size) or {} + return f"{z['cpus']} vCPU, {z['memory_gb']} GB" if "cpus" in z and "memory_gb" in z else "" + + +def fits(size: str, vm_memory_gb) -> bool: + """Whether a VM of ``size`` can boot a task's own VM of ``vm_memory_gb`` (OSWorld's guest): the guest's + memory must be free inside it, so the size needs more than the guest; a size of unmeasured memory, or + no guest, is not refused here.""" + have = (SIZES.get(size) or {}).get("memory_gb") + return not vm_memory_gb or not isinstance(have, (int, float)) or have > vm_memory_gb + + def size_problem(client: Client, size: str) -> Optional[str]: """Why this account cannot start a VM of ``size``, or None.""" if size not in SIZES: @@ -274,7 +291,7 @@ class VM: auto-stop, asked for; ``self.ttl`` the one it has, ``TTL`` on a free trial.""" def __init__(self, client: Client, keys: Keys, job: str, *, size: str = SIZE, stopped: Optional[threading.Event] = None, - wait: float = WAIT_SECONDS, sleep: Callable[[float], None] = time.sleep, ttl: int = TTL): + wait: float = WAIT_SECONDS, sleep: Callable[[float], None] = time.sleep, ttl: int = TTL, lock: bool = True): self.client, self.keys, self.id, self.size, self.ttl = client, keys, None, size, max(int(ttl), TTL) self.stopped = stopped t0 = time.time() @@ -296,7 +313,8 @@ def __init__(self, client: Client, keys: Keys, job: str, *, size: str = SIZE, st self.id = d["sandbox"]["id"] try: self._ready(sleep) - self.sh(f"sudo -n useradd -m -s /bin/bash {USER} && " + LOCK.format(user=USER)) + if lock: # a whole unit (boat_units) is the run's own code: no agent's user to lock in + self.sh(f"sudo -n useradd -m -s /bin/bash {USER} && " + LOCK.format(user=USER)) except BaseException: self.delete() raise diff --git a/app/nullius/standard/boat_units.py b/app/nullius/standard/boat_units.py new file mode 100644 index 00000000..6dfd7fd0 --- /dev/null +++ b/app/nullius/standard/boat_units.py @@ -0,0 +1,212 @@ +"""A benchmark whose units boot a VM of their own (``definitions.boots_vms``: OSWorld) on Boat: each +unit whole in a Boat VM of its own. + +The ``boat`` backend moves only an agent's process into a VM; such a unit's +weight is elsewhere - its driver, the upstream's script and the VM they boot +on the driver's machine. So here the Boat VM is that machine, for one unit: +this runner's own snapshot (``payload``) is sent to it and run there as a +shard of one unit on local processes (``shard``, by ``position``: shard I of +N, N the dataset's units, is the unit at place I-1), which installs what it +needs itself - the upstream, the driver's environment, the VM image - as on +any machine. The Boat image has the rest (Docker for its user, KVM, git, +uv); what the benchmark's chosen options still need of a machine is +installed with the VM's sudo. Each shard's job comes back to this machine as +its unit ends, the VM is deleted, and the shards are merged here into the +one job the run is named for (``merge``), recorded as a Boat run. + +As many VMs at once as the account runs (``boat.slots``), never more than +the run's own concurrency; a VM the account cannot start yet is waited for. +Every VM carries the run's label, so a Stop or a crash leaves none +(``boat.sweep``). The VMs are the run's own code: no locked agent's user +(``boat.VM``, ``lock``). Each model-calling unit uses a separate relay container in its Boat VM. +""" +from __future__ import annotations + +import os +import json +import shlex +import shutil +import subprocess +import threading +import uuid +from typing import Callable, Dict, List, Optional + +from nullius import paths, secrets +from nullius.standard import boat, definitions, hostreq, merge, payload, shard + +RUNPY = "import json, os, runpy, sys; os.environ.update(json.loads(sys.stdin.readline())); sys.path.insert(0, sys.argv.pop(1)); runpy.run_module('nullius.standard.cli', run_name='__main__', alter_sys=True)" +SHARDS = ".boat" # the shards' jobs, under the jobs root, until they are merged + + +class Refused(Exception): + """The run cannot go this way; the message says why.""" + + +def plan(bench: dict, dataset: str, limit: Optional[int]) -> tuple: + """(N, the shards to run): N the dataset's units as its definition declares them, so each shard + is one unit; ``limit`` runs the first ones.""" + if limit is not None and (not isinstance(limit, int) or limit < 1): + raise Refused("--limit must be a positive whole number") + ds = next((d for d in bench["datasets"] if d["id"] == dataset), None) + if ds is None: + raise Refused(f"{bench['label']} has no dataset {dataset!r}") + n = ds.get("units") + if not isinstance(n, int) or n < 1: + raise Refused(f"{bench['label']}'s dataset {dataset} declares no number of units, which a Boat run " + "of one VM per unit needs") + return n, list(range(1, min(limit or n, n) + 1)) + + +def child_argv(words: List[str], *, app: str, index: int, n: int, launch: str, name: str, jobs: str) -> str: + """The shell line a VM runs: this runner's snapshot, the unit's shard on local processes.""" + argv = ["python3", "-I", "-S", "-B", "-c", RUNPY, app + "/app", "run", *words, + "--shard", f"{index}/{n}", "--shard-by", "position", "--launch", launch, + "--name", name, "--jobs", jobs, "-j", "1", "--from-run", "--unsandboxed", "--relay-container"] + env = {"NULLIUS_PROGRESS": "1", "PYTHONDONTWRITEBYTECODE": "1", "NULLIUS_CATALOGUE": app + "/catalogue"} + return ("export PATH=\"$HOME/.local/bin:$PATH\" " + " ".join(f"{k}={shlex.quote(v)}" for k, v in env.items()) + + " && " + " ".join(shlex.quote(w) for w in argv)) + + +def prepare(vm: "boat.VM", bench: dict, needs: List[str]) -> None: + """What the chosen options need of the machine that Boat's image lacks, installed with its sudo.""" + lines = hostreq.install_lines(bench, needs, sudo="sudo -n ") + if lines: + vm.sh("export DEBIAN_FRONTEND=noninteractive && sudo -n apt-get update -q >/dev/null && " + + " && ".join(c.strip() for c in lines)) + + +def run(bench: dict, words: List[str], *, dataset: str, limit: Optional[int], concurrency: int, jobs_root: str, + name: str, launch: str, size: str, needs: List[str], say: Callable[[str], None], + credentials: Optional[dict] = None, env=os.environ, client: Optional["boat.Client"] = None, stopped: Optional[threading.Event] = None) -> tuple: + """(the merged job's folder, whether it failed): every unit run whole in a Boat VM of its own.""" + why = boat.unusable(env=env) or boat.size_problem(client or boat.Client(boat.key(env) or ""), size) + if why: + raise Refused(f"Boat cannot run it: {why}") + client = client or boat.Client(boat.key(env) or "") + n, wanted = plan(bench, dataset, limit) + jobs_root = os.path.abspath(jobs_root) + out = os.path.join(jobs_root, name) + if os.path.exists(out): + raise Refused(f"{out} exists: a run's name is used once") + here = os.path.join(jobs_root, SHARDS, name) + most = boat.slots(client) or 1 + at_once = max(1, min(concurrency or 1, most, len(wanted))) + say(f"Boat: {len(wanted)} {bench['units']['nouns'] if len(wanted) > 1 else bench['units']['noun']}, " + f"each in a {boat.SIZES[size]['label']} VM of its own, {at_once} at a time") + starts = boat.starts_problem(client, len(wanted)) + if starts: + say(starts) + _, app_here = payload.build(os.path.join(paths.cache_dir(), "standard-app")) + app = "/home/user/.nullius/app/" + os.path.basename(app_here) # where the VM keeps it, apart from this machine's paths + label = f"{name}.{uuid.uuid4().hex[:8]}" + stopped = stopped or threading.Event() + redactions = list((credentials or {}).values()) + redactions += [v.decode(errors="replace") for v in secrets.unpack((credentials or {}).get(secrets.HANDED, "")).values()] + keys = boat.Keys() + queue = list(wanted) + lock = threading.Lock() + procs: Dict[int, subprocess.Popen] = {} + done: Dict[int, int] = {} + + def one(i: int) -> None: + sname = f"{name}__shard-{i}of{n}" + vm = None + try: + vm = boat.VM(client, keys, label, size=size, stopped=stopped, lock=False) + say(f"[{i}/{n}] VM {vm.id} ready in {vm.seconds} s") + # the shard's job at the path it has here, so it comes back by the same path (VM.fetch) + vm.sh(f"mkdir -p {shlex.quote(os.path.dirname(app))} && sudo -n mkdir -p {shlex.quote(here)} && " + f"sudo -n chown \"$(id -u):$(id -g)\" {shlex.quote(here)}") + # the snapshot, under the VM user's home: tar from here, untarred there + tar = subprocess.Popen(["tar", "-C", os.path.dirname(app_here), "-czf", "-", os.path.basename(app_here)], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + try: + vm.sh(f"tar -C {shlex.quote(os.path.dirname(app))} -xzf - && rm -rf {shlex.quote(app + '/catalogue')} && " + f"cp -R {shlex.quote(app + '/app/nullius/catalogue')} {shlex.quote(app + '/catalogue')}", stdin=tar.stdout) + finally: + tar.stdout.close() + tar.wait() + prepare(vm, bench, needs) + if stopped.is_set(): + return + line = child_argv(words, app=app, index=i, n=n, launch=launch, name=sname, jobs=here) + p = subprocess.Popen(vm.ssh + ["-T", vm.target, line], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, errors="replace") + with lock: + procs[i] = p + try: + p.stdin.write(json.dumps(credentials or {}) + "\n") + p.stdin.close() + except BrokenPipeError: + pass + for ln in p.stdout: + ln = ln.rstrip() + if ln: + for secret in redactions: + if secret: + ln = ln.replace(secret, "[redacted]") + say(f"[{i}/{n}] {ln}") + rc = p.wait() + os.makedirs(os.path.join(here, sname), exist_ok=True) + vm.fetch(os.path.join(here, sname)) + with lock: + done[i] = rc + except boat.Unavailable as e: + say(f"[{i}/{n}] not run: {e}") + finally: + with lock: + procs.pop(i, None) + if vm is not None: + left = vm.delete() + if left: + say(f"[{i}/{n}] Boat would not delete VM {vm.id}: {left}") + + def worker() -> None: + while not stopped.is_set(): + with lock: + if not queue: + return + i = queue.pop(0) + one(i) + + threads = [threading.Thread(target=worker, daemon=True) for _ in range(at_once)] + try: + for t in threads: + t.start() + for t in threads: + while t.is_alive(): + t.join(1) + except KeyboardInterrupt: + stopped.set() + with lock: + for p in procs.values(): + p.terminate() + for t in threads: + t.join(60) + raise + finally: + boat.sweep(label, client, say) + keys.close() + shards = sorted(os.path.join(here, d) for d in os.listdir(here)) if os.path.isdir(here) else [] + shards = [d for d in shards if os.path.isfile(os.path.join(d, "config.json"))] + if not shards: + raise Refused("no unit came back from its Boat VM") + got = merge.merge(shards, out, force=False) + for note in got.get("notes") or []: + if not note.startswith("shard"): # a shard is a unit here: the units not run are said once, below + say(note) + _record(out, size) + shutil.rmtree(here, ignore_errors=True) + failed = not done or all(rc not in (0,) for rc in done.values()) + return out, failed + + +def _record(job_dir: str, size: str) -> None: + """The merged job says where its units ran: Boat, the VMs' size; its shards ran on local processes there.""" + path = os.path.join(job_dir, "config.json") + with open(path) as f: + cfg = json.load(f) + cfg.setdefault("standard", {}).update(backend="boat", boat_size=size, boat_units="each unit whole in a Boat VM of its own") + with open(path + ".tmp", "w") as f: + json.dump(cfg, f, indent=2) + os.replace(path + ".tmp", path) diff --git a/app/nullius/standard/cli.py b/app/nullius/standard/cli.py index b4aaafb3..4698f7a3 100644 --- a/app/nullius/standard/cli.py +++ b/app/nullius/standard/cli.py @@ -73,6 +73,7 @@ import signal import sys import tempfile +import time from nullius.standard.retries import MAX_RETRIES, RETRIES from nullius import constraints as limits, paths, prices as price_list, progress, providers, secrets, settings @@ -545,6 +546,8 @@ def cmd_run(a): if why: print("refused: constraints:", why) return 2 + if a.backend == "boat" and definitions.boots_vms(b): + return _boat_units(a, b, other) # a cap counts each call at its model's price: the public list fetched first (Settings, Model prices); a machine # that cannot reach it runs uncapped where a model has no price, and says so (run.Run) unreachable = None @@ -557,8 +560,8 @@ def cmd_run(a): try: job, failed = run.run_and_report(b, other, model=a.model or "", provider=a.provider, dataset=a.dataset, limit=a.limit, concurrency=a.concurrency, jobs_root=a.jobs, name=a.name, unsandboxed=a.unsandboxed, judge_model=a.judge_model, backend=a.backend, - boat_size=a.boat_size, simulator_model=a.simulator_model, options=options(a.option), shard=_shard(a.shard), launch=a.launch, - licences=held, bench_options=options(a.bench_option), repeat=a.repeat, retries=a.retries, + boat_size=a.boat_size, relay_container=a.relay_container, simulator_model=a.simulator_model, options=options(a.option), shard=_shard(a.shard), launch=a.launch, + shard_by=a.shard_by, licences=held, bench_options=options(a.bench_option), repeat=a.repeat, retries=a.retries, remove_images=a.remove_images, prices=prices(a.price), max_usd=a.max_usd, max_usd_per_unit=a.max_usd_per_unit, prices_unreachable=unreachable, constraints=given, @@ -579,6 +582,94 @@ def cmd_run(a): return 3 if failed else 0 +def boat_credentials(a, bench, other, vals) -> dict: + """Only the selected provider and declared benchmark licences cross SSH stdin.""" + import base64 + needs = definitions.option_needs(bench, vals) + out = {} + if other["_kind"] != "control" or (bench.get("score") or {}).get("judge") or needs["embeddings"] or needs["reranker"]: + base, key = run.provider_endpoint(a.provider, run_env(a.provider) if a.from_run else os.environ) + out.update(OPENAI_API_KEY=key, OPENAI_BASE_URL=base) + held, spill = licences(bench, from_run=a.from_run) + try: + held = held or run.Licences(bench) + values = {} + for name in held.kinds: + item = held.get(name) + if item: + if item["kind"] == "file": + with open(item["path"], "rb") as f: + value = f.read() + else: + value = item["value"].encode() + values[name] = base64.b64encode(value).decode() + if values: + out[secrets.HANDED] = base64.b64encode(json.dumps(values).encode()).decode() + finally: + if spill: + shutil.rmtree(spill, ignore_errors=True) + return out + + +def _boat_units(a, b, other) -> int: + """A benchmark whose units boot VMs of their own, on Boat: each unit whole in a Boat VM (``boat_units``).""" + from nullius.standard import boat_units + if a.units or a.shard: + print("refused: whole-task Boat runs do not accept --units or --shard; choose a dataset and --limit") + return 2 + ds = a.dataset or b["datasets"][0]["id"] + vals, bad = definitions.bench_options_of(b, ds, options(a.bench_option), + a.other if other["_kind"] == "upstream" else "yours", + kind=other["_kind"]) + if bad: + print("refused:", "; ".join(bad)) + return 2 + words = [b["key"] if "_version" not in b else definitions.ref_of(b), a.other, "--provider", a.provider, + "--dataset", ds, "--attempts", str(a.repeat), "--retries", str(a.retries)] + for field in ("model", "judge_model", "simulator_model", "max_usd_per_unit", "max_turns", "max_calls"): + value = getattr(a, field) + if value is not None: + words += ["--" + field.replace("_", "-"), str(value)] + for field in ("bench_option", "option", "timeout", "constraint", "served_by", "price"): + for value in getattr(a, field) or []: + words += ["--" + field.replace("_", "-"), value] + # Saved price overrides belong to the launching profile, not the fresh VM. + needs = definitions.option_needs(b, vals) + models = [a.model, other.get("sent"), a.judge_model or providers.served_as((b.get("score", {}).get("judge") or {}).get("model", ""), a.provider), + a.simulator_model or providers.served_as((b.get("simulator") or {}).get("model", ""), a.provider), + *[providers.served_as(m, a.provider) for m in needs["embeddings"] + needs["reranker"]]] + explicit = prices(a.price) + for model, row in price_list.given(models, price_list.table()).items(): + if model not in explicit: + words += ["--price", model + "=" + ",".join(str(row.get(k, "")) for k in price_list.RATES).rstrip(",")] + if a.remove_images: + words.append("--remove-images") + try: + _, wanted = boat_units.plan(b, ds, a.limit) + if a.max_usd is not None: + if not 0 < a.max_usd < float("inf"): + raise boat_units.Refused("--max-usd must be finite and above zero") + # Independent VMs cannot share a ledger: each gets an equal, bounded share. + words += ["--max-usd", str(a.max_usd / len(wanted))] + creds = boat_credentials(a, b, other, vals) + except (run.Refused, boat_units.Refused) as e: + print("refused:", e) + return 2 + stamp = time.strftime("%Y-%m-%d__%H-%M-%S") + name = a.name or f"{stamp}__{definitions.job_word(b)}__{definitions.job_word(other)}" + try: + job_dir, failed = boat_units.run(b, words, dataset=a.dataset or b["datasets"][0]["id"], limit=a.limit, + concurrency=a.concurrency, jobs_root=a.jobs, name=name, size=a.boat_size, + launch=a.launch or shard.launch_id(definitions.job_word(b), definitions.job_word(other), ""), + needs=definitions.option_needs(b, vals)["host"], credentials=creds, say=lambda s: print(s, flush=True), + env={**os.environ, **run_env(a.provider)} if a.from_run else os.environ) + except (boat_units.Refused, boat.Unavailable, merge.Refused) as e: + print("refused:", e) + return 2 + print(job_dir) + return 3 if failed else 0 + + def _shard(spec): if not spec: return None @@ -704,6 +795,8 @@ def main(argv=None): help="local processes (the default); container: each unit in a container, its relay beside it " "(docker: its earlier name); boat: each unit's agent in a boat.dev VM of its own") if name == "run": + p.add_argument("--relay-container", action="store_true", + help="run the relay in a separate container while task processes run on this host") p.add_argument("--boat-size", default=boat.SIZE, choices=list(boat.SIZES), help="on the boat backend, its VMs' machine type (default small: 2 vCPU, 4 GB)") p.add_argument("--from-run", action="store_true", @@ -736,6 +829,9 @@ def main(argv=None): help="remove each image this run pulls or builds when its last unit ends (default: kept for the next run)") p.add_argument("--name") p.add_argument("--shard", metavar="I/N", help="only shard I of N of the units: by the unit id's hash, before --limit") + if name == "run": + p.add_argument("--shard-by", choices=list(shard.BY), default="id", + help="split by the unit id's hash (id, the default) or by its place in the dataset (position)") p.add_argument("--launch", help="the launch id every shard records (default: the day, the benchmark, the agent, the model)") p.add_argument("--bench-option", action="append", metavar="NAME=VALUE", help="one of the benchmark's own options (repeatable: a search, a mode); the rest keep their defaults") diff --git a/app/nullius/standard/definitions.py b/app/nullius/standard/definitions.py index 17b7f921..c2af34c1 100644 --- a/app/nullius/standard/definitions.py +++ b/app/nullius/standard/definitions.py @@ -517,9 +517,13 @@ def _bench_options(d: dict) -> List[str]: bad.append(f"{w}.needs: value -> {{embeddings, reranker, host, files}}") continue for v, n in needs.items(): - if option_problem(o, v) is not None or not isinstance(n, dict) or any(k not in NEED_KEYS + ("when",) for k in n): - bad.append(f"{w}.needs.{v}: one of its values -> {{embeddings: a model, reranker: a model, host: [requirements], files: [names]}}") + if option_problem(o, v) is not None or not isinstance(n, dict) or any(k not in NEED_KEYS + ("when", "vm") for k in n): + bad.append(f"{w}.needs.{v}: one of its values -> {{embeddings: a model, reranker: a model, host: [requirements], files: [names], vm: {{memory_gb}}}}") continue + if "vm" in n and (o.get("step") != "where" or not isinstance(n["vm"], dict) or set(n["vm"]) != {"memory_gb"} + or isinstance(n["vm"]["memory_gb"], bool) or not isinstance(n["vm"]["memory_gb"], (int, float)) + or n["vm"]["memory_gb"] <= 0): + bad.append(f"{w}.needs.{v}.vm: on a Where option, the memory in GB of the VM each unit boots: {{memory_gb: a number}}") if not _when_valid(name, n.get("when") or {}, opts): bad.append(f"{w}.needs.{v}.when: another option -> the values that need these requirements") for kind in ("embeddings", "reranker"): @@ -625,6 +629,21 @@ def option_needs(bench: dict, values: dict) -> dict: return out +def vm_memory_gb(bench: dict, values: dict) -> float: + """The memory in GB of the VM each unit boots of its own (OSWorld's guest) under the chosen option + values: a Where option's ``needs..vm``; 0 when it boots none.""" + opts = bench.get("options") or {} + return max([((opts.get(n, {}).get("needs") or {}).get(v) or {}).get("vm", {}).get("memory_gb", 0) + for n, v in values.items() if _when_applies(opts.get(n, {}).get("when") or {}, opts, values)] or [0]) + + +def boots_vms(bench: dict) -> bool: + """Whether each unit boots a VM of its own on the machine its driver runs on: a Where option of the + benchmark has a value that declares one (``needs..vm``).""" + return any("vm" in (n or {}) for o in (bench.get("options") or {}).values() if o.get("step") == "where" + for n in (o.get("needs") or {}).values()) + + def options_of(d: dict, given: Optional[dict] = None) -> Tuple[dict, List[str]]: """Every option's value for one run - the ones ``given``, the rest at their defaults - and what is wrong with the given ones (a name it does not have, diff --git a/app/nullius/standard/recipe.py b/app/nullius/standard/recipe.py index 018cba1b..4afbeeb3 100644 --- a/app/nullius/standard/recipe.py +++ b/app/nullius/standard/recipe.py @@ -547,7 +547,8 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, limit = recipe.get("limit") if limit not in ("", None) and _int(limit, 1) is None: add("limits", "The number of units is a whole number of at least 1, or empty for all of them.") - conc = _int(recipe.get("concurrency") or 1, 1, MAX_CONCURRENT) # empty: one at a time + # empty: one at a time; on Boat, as many as the account runs at once (the run caps it to that, boat.slots) + conc = _int(recipe.get("concurrency") or (MAX_CONCURRENT if backend == "boat" else 1), 1, MAX_CONCURRENT) if conc is None: add("limits", f"Units at a time: a whole number from 1 to {MAX_CONCURRENT}.") targets, why_not = where_of(recipe.get("where", "local")) @@ -557,7 +558,9 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, why = docker_unusable() if why: add("where", f"The container backend cannot run on this machine: {why}. Choose local processes, or a host with Docker or Podman.") - elif backend == "boat" and boat_unusable is not None and (why := boat_unusable("local" in targets)): + elif backend == "boat" and targets != ["local"]: + add("where", "Hosts and Boat sandboxes do not mix: choose Hosts, or Sandboxes alone.") + elif backend == "boat" and boat_unusable is not None and (why := boat_unusable(True)): add("where", f"Boat cannot run: {why}.") elif backend == "boat" and boat_size_problem is not None and str(recipe.get("boat_size") or boat.SIZE) in boat.SIZES: why = boat_size_problem(str(recipe.get("boat_size") or boat.SIZE)) @@ -567,6 +570,22 @@ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *, if backend == "boat" and boat_size not in boat.SIZES: add("where", f"Choose a Boat size: {', '.join(boat.SIZES)}.") boat_size = boat.SIZE + # a benchmark whose units boot a VM of their own (OSWorld's guest): the VM starts on the machine the + # driver runs on, which a container cannot do, and on Boat inside a VM big enough to hold it + if b and definitions.boots_vms(b) and backend == "container": + add("where", f"{b['label']} starts each unit's VM on the machine: choose local processes.") + guest = max([definitions.vm_memory_gb(b, v) for v in per_ds.values()] or [0]) + if backend == "boat" and not boat.fits(boat_size, guest): + add("where", f"A {boat.SIZES[boat_size]['label']} VM is too small for {b['label']}, whose units boot a {guest:g} GB VM: choose a larger size.") + # on Boat each unit's agent or capability runs in its VM (run.Run refuses the rest): a setup or a check runs + # on a machine, and the computer port's socket is on this one, out of a VM's reach + # A benchmark booting VMs of its own runs each unit whole in a VM (boat_units), with a relay container in each VM + computer = run.computer_port(b) + whole = definitions.boots_vms(b) + elsewhere = [definitions.setup_label(b, "original") if p.get("setup") else p["o"]["label"] for p in picks + if not whole and (p["control"] or p.get("setup") or (computer and side == "agent"))] + if backend == "boat" and elsewhere: + add("where", f"{', '.join(elsewhere)} cannot run in a Boat VM yet: choose Hosts.") # a job that calls no model (a control of a benchmark with no judge) spends nothing and takes no share spends = lambda p: not (p["control"] and not (b.get("score") or {}).get("judge") and not needs["embeddings"] and not needs["reranker"]) paying = len(dss) * sum(1 for p in picks if spends(p)) * len(targets or [None]) @@ -675,7 +694,7 @@ def command_for(d: dict, p: dict, index: int) -> str: *(["--judge-model", judge_model] if judge_model else []), "--name", job_of(d, p, index), *[w for k, v in given.items() for w in ("--option", f"{k}={v}")], *(["--backend", backend] if backend != "local" else []), - *(["--boat-size", boat_size] if backend == "boat" and not control and not p.get("setup") else []), + *(["--boat-size", boat_size] if backend == "boat" and (whole or not control and not p.get("setup")) else []), *(["--unsandboxed"] if side == "agent" and not control and not p.get("setup") and backend == "local" else []), *(["--shard", f"{index}/{of}", "--launch", launch_of(d, p)] if of > 1 else []), *limits.cli_words(limits.of(targets[index - 1])), @@ -718,7 +737,7 @@ def script_for(d: dict, p: dict, command: str, index: int) -> str: # Nothing in the run asks a model: no key is handed to it, and none need be set. out["keyless"] = only_controls and not (b.get("score") or {}).get("judge") and not needs["embeddings"] and not needs["reranker"] # what the machines must have for the chosen options (``host``): the Where step and its test show it, Run checks it - on_host = needs["host"] + on_host = [] if backend == "boat" and whole else needs["host"] # each VM installs them (boat_units.prepare) if backend == "container" and any(p.get("setup") for p in picks): # a container setup's image installs what has an install; nothing else reaches its container imaged = hostreq.in_image(b, on_host) @@ -766,7 +785,8 @@ def row(p: dict) -> list: + (f" · judge {judge_model or b['score']['judge']['model']}" + ("" if judge_model else " (official)") if (b.get("score") or {}).get("judge") else "") + "".join(f" · {SERVED_LABEL[r_]} served by {', '.join(s)}" for r_, s in served.items())], [STEP["limits"], "Scope", f"{'first ' + str(n) if isinstance(limit, int) else 'all ' + str(total or '')} {nouns}" - + (" of each shard" if of > 1 and isinstance(limit, int) else "") + f" · {conc} at a time".replace(" ", " ") + + (" of each shard" if of > 1 and isinstance(limit, int) else "") + + (" · as many at a time as Boat allows" if backend == "boat" and not recipe.get("concurrency") else f" · {conc} at a time") + (" of each dataset" if many and isinstance(limit, int) else "") + (f" · {len(pairs)} jobs, one after the other" if len(pairs) > 1 else "") + (f" · {repeat} attempts each (pass^1 to pass^{repeat})" if repeat > 1 else "") @@ -779,9 +799,9 @@ def row(p: dict) -> list: *([[STEP["limits"], "Time limits", " · ".join(f"{k} {v / 60:g} min" for k, v in forced.items())]] if forced else []), *([[STEP["limits"], "Turns per conversation", str(turns)]] if turns else []), *([[STEP["limits"], f"Model calls per {noun}", str(calls)]] if calls else []), - [STEP["where"], "Where", (f"{of} shards, one per machine: " if of > 1 else "") - + ", ".join(where_label(t) for t in targets) + f" · {backend} backend" - + (f" · {boat_size} VMs" if backend == "boat" else "") + [STEP["where"], "Where", (f"Boat sandboxes · one {boat.SIZES[boat_size]['label']} VM per {noun}" if backend == "boat" + else (f"{of} shards, one per machine: " if of > 1 else "") + + ", ".join(where_label(t) for t in targets) + f" · {backend} backend") + "".join(f" · {w.get('label') or k.replace('_', ' ').capitalize()} {(w.get('names') or {}).get(v) or v}" for k, v in {k: v for vals in per_ds.values() for k, v in vals.items()}.items() if (w := (b.get("options") or {})[k]).get("step") == "where")]] if agents_local: diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 4b17e9b3..8a519c80 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -460,7 +460,7 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas limit: Optional[int] = None, concurrency: int = 4, jobs_root: str = "jobs", name: Optional[str] = None, turn_seconds: Optional[int] = None, say: Callable[[str], None] = print, env=os.environ, proving: bool = False, unsandboxed: bool = False, judge_model: Optional[str] = None, - backend: str = "local", options: Optional[dict] = None, shard: Optional[tuple] = None, + backend: str = "local", options: Optional[dict] = None, shard: Optional[tuple] = None, shard_by: str = "id", launch: Optional[str] = None, repeat: int = 1, retries: int = RETRIES, retry_if: Optional[Callable[[dict], bool]] = None, simulator_model: Optional[str] = None, licences: Optional[Licences] = None, bench_options: Optional[dict] = None, which=shutil.which, @@ -468,20 +468,23 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas max_usd_per_unit: Optional[float] = None, prices_unreachable: Optional[str] = None, units_file: Optional[dict] = None, constraints: Optional[dict] = None, timeouts: Optional[dict] = None, max_turns: Optional[int] = None, - max_calls: Optional[int] = None, served_by: Optional[Dict[str, List[str]]] = None, boat_size: str = boat.SIZE): + max_calls: Optional[int] = None, served_by: Optional[Dict[str, List[str]]] = None, boat_size: str = boat.SIZE, relay_container: bool = False): """``proving``: the run is a conformance kit's, the one run that may start before the port it checks is proven - the family must still match. ``unsandboxed``: consent to execute code as the user: required for local agents, and for setup checks that run code without Docker. ``backend``: ``local``, ``container`` (``docker``, its earlier name), which needs no such yes and is refused when docker is missing or not usable, or ``boat``; + ``relay_container``: keep local task processes on the host but run each + unit's relay in a separate container, as inside a whole-task Boat VM. ``boat_size``: its VMs' machine type (``boat.SIZES``), refused when the account cannot start it. ``options``: the agent's or capability's options set for this run, by name; the rest keep their defaults, and the job's config records every value. ``shard``: (I, N), only the units of shard I of N (``shard``), before - ``limit``; ``launch``: the id every shard of one launch records. + ``limit``, ``shard_by`` its units' id (by hash) or place (``shard.BY``); ``launch``: the id every + shard of one launch records. ``units_file``: the units a file lists (:func:`units_file`): only those of the dataset run, in the dataset's order, before ``limit``; an id the dataset does not hold refuses the run before any unit @@ -541,7 +544,9 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas if backend not in BACKENDS: raise Refused(f"backend {backend!r} is not one of {list(BACKENDS)}") if backend == "boat" and other.get("_kind") not in ("agent", "capability"): - backend = "local" # a check or a setup: nothing of an agent's runs, so nothing goes to a VM + # a setup or a check runs on the machine; on Boat it would run here unasked, beside the VMs + raise Refused(f"{other['label']} runs on a machine, not in a Boat VM: run it on this machine or a host " + "(--backend local or container)") if backend == "boat" and other.get("_kind") == "agent" and computer_port(bench): raise BenchmarkRefused("the computer port uses a socket on this machine, which an agent in a Boat VM " "cannot reach: choose local processes or the container backend") @@ -557,7 +562,10 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas why = [w for w in why if "does not offer" not in w and "has not passed its kit" not in w] if why: raise Refused(f"{other['label']} cannot run {bench['label']}: " + "; ".join(why)) - if backend == "container": + self.relay_container = bool(relay_container) + if relay_container and backend != "local": + raise Refused("a separate relay container requires the local process backend") + if backend == "container" or relay_container: why = container.unusable() if why: raise Refused(f"the container backend cannot run here: {why}") @@ -740,6 +748,9 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas self.say = say stamp = time.strftime("%Y-%m-%d__%H-%M-%S") self.shard = tuple(shard) if shard and shard[1] > 1 else None + if shard_by not in shards.BY: + raise Refused(f"--shard-by {shard_by!r}: one of {', '.join(shards.BY)}") + self.shard_by = shard_by self.launch = launch or (shards.launch_id(definitions.job_word(bench), definitions.job_word(other), model) if self.shard else None) self.name = name or (f"{stamp}__{definitions.job_word(bench)}__{definitions.job_word(other)}" @@ -834,7 +845,7 @@ def _start(self, lck, mine) -> str: HELD[lck] = held self.say(f"job {self.job_dir}") p, control = self.progress, self.other["_kind"] == "control" - p.emit("plan", stages=",".join((["image"] if self.backend == "container" or self.container_check else []) + p.emit("plan", stages=",".join((["image"] if self.backend == "container" or self.relay_container or self.container_check else []) + (["upstream"] if install.upstream_dir(self.bench) else []) + (["dataset"] if self.ds.get("url") else []) + (["driver"] if install.driver_dir(self.bench) else []) @@ -859,7 +870,7 @@ def _start(self, lck, mine) -> str: driver.close() if self.bench_relay is not None: self.bench_relay.stop() - if self.backend == "container": + if self.backend == "container" or self.relay_container: container.sweep(self.name) raise everything = [str(u) for u in units] @@ -874,7 +885,7 @@ def _start(self, lck, mine) -> str: d.close() if self.bench_relay is not None: self.bench_relay.stop() - if self.backend == "container": + if self.backend == "container" or self.relay_container: container.sweep(self.name) # the record a refusal in ready() leaves, which the runs list reads (execution's refused): # the job's folder with its progress.jsonl, the driver's working folders and logs taken out @@ -891,7 +902,7 @@ def _start(self, lck, mine) -> str: p.emit("refused", detail=why) raise Refused(why) # the list and the shard, then the limit; each unit keeps its place in the whole dataset (its trial's tag) - mine = shards.select(everything, *self.shard) if self.shard else list(enumerate(everything)) + mine = shards.select(everything, *self.shard, by=self.shard_by) if self.shard else list(enumerate(everything)) if listed is not None: wanted = set(listed) mine = [(i, u) for i, u in mine if u in wanted] @@ -911,7 +922,7 @@ def _start(self, lck, mine) -> str: image=self.setup_driver.image if self.setup_driver else self.image, options=self.options, launch=self.launch or "", bench_options=self.bench_options, # a shard of a list splits the listed units: they are its whole - shard=shards.record(*self.shard, len(everything) if listed is None else len(listed), units, self.limit) + shard=shards.record(*self.shard, len(everything) if listed is None else len(listed), units, self.limit, by=self.shard_by) if self.shard else None, units_file=self.units_file, repeat=self.repeat, retries=self.retries, prices=self.prices_record(), constraints=self.constraints, served_by=self.serving, timeouts=self.timeouts, max_turns=self.forced_turns, max_calls=self.max_calls, @@ -957,7 +968,7 @@ def work_unit(iur): futures = [] try: # a model server on this machine's loopback, which the relays' containers cannot reach as written - if self.backend == "container" and self.base and container.on_loopback(self.base): + if (self.backend == "container" or self.relay_container) and self.base and container.on_loopback(self.base): self.loopback = container.Loopback(self.base, image=self.image, job=self.name).start() for iur in work: futures.append(pool.submit(work_unit, iur)) @@ -973,7 +984,7 @@ def work_unit(iur): except BaseException: self._stopped.set() self._halt.set() - if self.backend == "container": + if self.backend == "container" or self.relay_container: container.sweep(self.name) raise finally: @@ -986,7 +997,7 @@ def work_unit(iur): d.close() if self.bench_relay is not None: self.bench_relay.stop() - if self.backend == "container": + if self.backend == "container" or self.relay_container: container.sweep(self.name) # whatever an interrupted unit left if getattr(self, "boat", None) is not None: self.boat.close() @@ -1170,7 +1181,7 @@ def ready(self) -> None: p = self.progress self.scope_limits = self.scope_ready(limits.scoped(self.constraints), refuse=self.backend == "local") owns = any(definitions.own_image(e) for e in (self.bench.get("environments") or {}).values()) - if self.backend == "container" or self.container_check: + if self.backend == "container" or self.relay_container or self.container_check: try: tag = container.base_tag() there = container.present(tag) @@ -1180,7 +1191,7 @@ def ready(self) -> None: raise Refused(f"the Docker image: {e}") # every container the run starts takes the caps: its agents', relays' and installs' on the container # backend, its checks' and own images' on either - if self.backend == "container" or self.container_check or owns: + if self.backend == "container" or self.relay_container or self.container_check or owns: refusal = Refused if self.backend == "container" else BenchmarkRefused try: # Docker can drop a limit with only a warning: a result would look limited when it was not @@ -1225,7 +1236,7 @@ def ready(self) -> None: with p.stage("driver", cached=install.ready(drv), start=f"installing {self.bench['key']}'s driver environment"): install.driver_packages(self.bench) except (install.InstallFailed, container.Unavailable) as e: - if self.backend == "container": + if self.backend == "container" or self.relay_container: container.sweep(self.name) raise BenchmarkRefused(f"could not install what the run needs: {e}") try: @@ -1234,7 +1245,7 @@ def ready(self) -> None: with p.stage("install", cached=install.ready(install.packages_dir(self.other, image=image)), agent=self.other.get("_ref") or self.other["key"]): self.inst = install.packages(self.other, run_in=run_in, image=image) except (install.InstallFailed, container.Unavailable) as e: - if self.backend == "container": + if self.backend == "container" or self.relay_container: container.sweep(self.name) raise Refused(f"could not install what the run needs: {e}") # An own image has none of what the agent runs on: it is mounted in, fetched once on this machine. @@ -1781,7 +1792,7 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] record = os.path.join(trial, ".work", "relay") side = tuple((self.other.get("model") or {}).get("side_models") or ()) key_header = self.key_header - if self.backend in ("local", "boat"): # boat: the agent reaches it through its VM's tunnel + if self.backend in ("local", "boat") and not getattr(self, "relay_container", False): # boat: the agent reaches it through its VM's tunnel rl = relay.Relay(self.base, self.key, record, prices=getattr(self, "prices", None), model=self.model, side_models=side, key_header=key_header, ledger=self.ledger_file(), serving=getattr(self, "serving", None), timeout=self.call_seconds, max_calls=self.max_calls).start() @@ -1790,6 +1801,16 @@ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] routes = {tag + "-judge": self.judge_model} if self.judge_model else {} if self.simulator_model: routes[tag + "-simulator"] = self.simulator_model + if getattr(self, "relay_container", False): + door = getattr(self, "loopback", None) + rl = container.Unit(self.name, tag, record, image=self.image, app=APP, + base=door.base if door else self.base, key=self.key, model=self.model, + routes=routes, side_models=side, key_header=key_header, + prices=price_list.given(self.models_called(), getattr(self, "prices", None)), + hosts=door.hosts if door else None, ledger=self.ledger_file(), + serving=self.serving, max_calls=self.max_calls, timeout=self.call_seconds).start() + rl.unit_url = rl.url(tag) # driver and agent run on this host, outside the relay container + return rl doors, listeners = [], [] if self.other["_kind"] == "agent" and mcp_machine.wanted(setup): # the benchmark's own servers run here, in the driver's environment; the agent reaches each diff --git a/app/nullius/standard/shard.py b/app/nullius/standard/shard.py index 476c5ddd..c8a6caa1 100644 --- a/app/nullius/standard/shard.py +++ b/app/nullius/standard/shard.py @@ -18,6 +18,11 @@ the 43rd unit in every shard), so the shards' trials never share a name and merge without renaming (``merge``). +A launch where each machine takes one unit (a Boat VM per unit, ``boat_units``) +splits by place instead (``BY``, ``position``): shard ``I`` of ``N`` holds the +units at places ``I-1``, ``I-1+N``, ... of the dataset's order, so ``N`` equal to +the dataset's units gives each shard exactly one. + Every shard of one launch records the same ``launch`` id: ``--launch``, or by default the day (UTC), the benchmark, the agent or capability and the model, so shards started separately - on other machines, minutes apart - still say @@ -32,6 +37,7 @@ from typing import List, Optional, Tuple PARTITION = "sha256(unit id) mod N" +BY = {"id": PARTITION, "position": "place in the dataset mod N"} _SPEC = re.compile(r"^\s*(\d+)\s*/\s*(\d+)\s*$") _WORD = re.compile(r"[^A-Za-z0-9_.-]+") @@ -52,10 +58,11 @@ def of(unit: str, n: int) -> int: return int.from_bytes(hashlib.sha256(str(unit).encode("utf-8")).digest()[:8], "big") % n + 1 -def select(units: List[str], index: int, n: int) -> List[Tuple[int, str]]: +def select(units: List[str], index: int, n: int, by: str = "id") -> List[Tuple[int, str]]: """The units of shard ``index`` of ``n``, each with its place in the whole - list (what its trial's tag is made of), in the dataset's order.""" - return [(i, u) for i, u in enumerate(units) if n == 1 or of(u, n) == index] + list (what its trial's tag is made of), in the dataset's order; ``by`` the + unit's id (its hash) or its place (``BY``).""" + return [(i, u) for i, u in enumerate(units) if n == 1 or (i % n + 1 if by == "position" else of(u, n)) == index] def launch_id(bench_word: str, other_word: str, model: str, day: Optional[str] = None) -> str: @@ -65,9 +72,9 @@ def launch_id(bench_word: str, other_word: str, model: str, day: Optional[str] = return "__".join(_WORD.sub("_", w).strip("_") for w in words) -def record(index: int, n: int, total: int, planned: List[str], limit: Optional[int] = None) -> dict: +def record(index: int, n: int, total: int, planned: List[str], limit: Optional[int] = None, by: str = "id") -> dict: """What a shard's config says of it (``standard.shard``): which shard, of how many, over how many units, split how; the units it planned to run, so a merge can name a unit planned and never run; and its own ``limit``.""" - return {"index": index, "of": n, "units_total": total, "partition": PARTITION, "planned": list(planned), + return {"index": index, "of": n, "units_total": total, "partition": BY[by], "planned": list(planned), **({"limit": limit} if limit else {})} diff --git a/app/static/builder/command-builder.css b/app/static/builder/command-builder.css index 8fd7695d..769c48da 100644 --- a/app/static/builder/command-builder.css +++ b/app/static/builder/command-builder.css @@ -188,9 +188,17 @@ #nullius-builder .nb-pick-none { color: var(--nb-muted); font-size: 13px; margin: 0; } @media (max-width: 640px) { #nullius-builder .nb-pick-chips + .nb-pick-chips { padding-left: 0; border-left: 0; } #nullius-builder .nb-pick-find { max-width: none; } #nullius-builder .nb-pick-find input { font-size: 16px; } } /* The Where step's machine setup: each chosen machine, its state for the search's requirements, check and install. */ -#nullius-builder .nb-hostreq { list-style: none; margin: 0 0 12px; padding: 0; display: grid; gap: 8px; } -#nullius-builder .nb-hostreq li { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; padding: 10px 12px; border: 1px solid var(--nb-line); border-radius: 8px; font-size: 12.5px; } -#nullius-builder .nb-hostreq small { color: var(--nb-muted); } +#nullius-builder .std-hostreq { padding: 16px; border: 1px solid var(--nb-line); border-radius: 8px; font-size: 13px; } +#nullius-builder .std-hostreq-row { display: flex; flex-wrap: wrap; align-items: center; gap: 12px; } +#nullius-builder .std-hostreq-row + .std-hostreq-row { margin-top: 12px; } +#nullius-builder .std-hostreq-names { flex: 1 1 240px; } +#nullius-builder .std-hostreq-actions { display: flex; flex-wrap: wrap; align-items: center; gap: 12px; } +#nullius-builder .std-hostreq small { color: var(--nb-muted); } +#nullius-builder .std-hostreq details { margin-top: 12px; } +#nullius-builder .std-hostreq summary { color: var(--nb-muted); cursor: pointer; } +#nullius-builder .std-hostreq .std-code { margin: 12px 0 0; display: flex; align-items: center; } +#nullius-builder .std-hostreq .std-code-head { order: 1; border: 0; } +#nullius-builder .std-hostreq .std-code pre { flex: 1; min-width: 0; overflow-wrap: anywhere; } #nullius-builder .std-code { border: 1px solid var(--nb-line); border-radius: 8px; background: var(--nb-bg); overflow: hidden; margin-bottom: 12px; } #nullius-builder .std-code-head { display: flex; justify-content: space-between; align-items: center; gap: 8px; padding: 5px 10px; border-bottom: 1px solid var(--nb-line); font: 11.5px var(--mono); color: var(--nb-muted); } #nullius-builder .std-copy { border: 1px solid var(--nb-line); background: var(--nb-surface); color: var(--nb-muted); border-radius: 6px; padding: 2px 9px; font-size: 11.5px; cursor: pointer; } @@ -250,7 +258,8 @@ #nullius-builder .std-backends .nb-tile-note { font-size: 11px; padding-left: 30px; } #nullius-builder .std-boat-sizes { grid-template-columns: repeat(4, minmax(0, 160px)); grid-auto-rows: auto; } #nullius-builder .std-boat-sizes .nb-tile { flex-direction: column; align-items: stretch; gap: 2px; } -#nullius-builder .std-boat-sizes .nb-tile-note { white-space: normal; } +#nullius-builder .std-boat-sizes .nb-tile-note { white-space: normal; padding-left: 0; } +#nullius-builder .std-boat-sizes .nb-tile:not([disabled]) .nb-tile-note { color: var(--nb-muted); } @media (max-width: 760px) { #nullius-builder .std-boat-sizes { grid-template-columns: repeat(2, minmax(0, 180px)); } } @media (max-width: 560px) { #nullius-builder .std-backends { grid-template-columns: minmax(0, 320px); } #nullius-builder .std-boat-sizes { grid-template-columns: repeat(2, minmax(0, 1fr)); } } /* Served by: a row after its model's row, as that row is: the label left, the picker right, the providers chosen @@ -265,3 +274,20 @@ @media (max-width: 650px) { #nullius-builder .std-served-list { grid-column: 1; } #nullius-builder label.std-served:has(.std-served-list:not(:empty)) > .ui-field-copy { grid-row: auto; margin-top: 0; } } /* a limit the benchmark or the agent declares: its placeholder reads as a value; Nullius's default stays grey */ #nullius-builder .nb-field input.nb-declared::placeholder { color: var(--nb-text); opacity: 1; } + +/* Where: machine kind, its machines, then isolation. Fixed values are not controls. */ +#nullius-builder .std-machines { display: inline-flex; gap: 3px; padding: 3px; border: 1px solid var(--nb-line); border-radius: 999px; } +#nullius-builder .std-machines button { padding: 6px 16px; border: 0; border-radius: 999px; background: transparent; color: var(--nb-text); font: 14px/1.4 var(--sans); } +#nullius-builder .std-machines button[aria-pressed=true] { background: var(--nb-tint); } +#nullius-builder .std-machines button:focus-visible { outline: 2px solid var(--nb-blue); outline-offset: 2px; } +#nullius-builder .std-machines button:disabled { opacity: .45; cursor: not-allowed; } +#nullius-builder .std-boat-card { display: flex; flex-direction: column; align-items: flex-start; gap: 6px; width: fit-content; max-width: 100%; min-width: 190px; margin-top: 12px; padding: 12px 14px; border: 1px solid var(--nb-blue); border-radius: 10px; background: var(--nb-tint); } +#nullius-builder .std-boat-card strong { font-size: 15px; font-weight: 500; } +#nullius-builder .std-boat-card > span { font-size: 13px; color: var(--nb-muted); } +#nullius-builder .std-boat-slots { border: 1px solid var(--nb-line); border-radius: 999px; padding: 2px 8px; } +#nullius-builder .std-hosts { margin-top: 12px; grid-template-columns: repeat(auto-fill, minmax(190px, 240px)); } +#nullius-builder .std-isolation { padding: 12px 14px; border: 1px solid var(--nb-line); border-radius: 10px; background: var(--nb-soft); } +#nullius-builder .std-isolation small { display: block; margin-top: 4px; font-size: 13px; color: var(--nb-muted); } +#nullius-builder .std-isolation small:empty { display: none; } +#nullius-builder .std-boat-sizes .nb-tile { min-height: 60px; } +#nullius-builder .std-boat-sizes .nb-tile-note { font-size: 13px; line-height: 1.4; } diff --git a/app/static/builder/command-builder.js b/app/static/builder/command-builder.js index 7f8d0045..83831868 100644 --- a/app/static/builder/command-builder.js +++ b/app/static/builder/command-builder.js @@ -457,42 +457,66 @@ return `${esc(name)}${proofSpan(p)}`; }; // The yes to an agent running as you: asked only for the local backend, where it is no sandbox. const consentBox = rec => ``; - // The standard's Where step: one machine or several (one shard each), and the backend on them. + // A benchmark whose units each boot a VM of their own (OSWorld's guest): a Where option has a value declaring + // one (needs..vm); vmMemory, that VM's memory in GB under the chosen values, 0 for none. + const bootsVms = b => Object.values((b || {}).options || {}).some(o => o.step === 'where' && Object.values(o.needs || {}).some(n => n && n.vm)); + const vmMemory = (rec, b) => Math.max(0, ...Object.entries((b || {}).options || {}).filter(([, o]) => o.step === 'where') + .map(([n, o]) => (((o.needs || {})[(rec.bench_options || {})[n] ?? o.default] || {}).vm || {}).memory_gb || 0)); + // Boat's sizes as the account offers them (nullius/standard/boat.py, SIZES), each with why it cannot be chosen: + // the account cannot start one, or it is too small to boot the unit's own VM (boat.fits). + const boatSizes = (rec, b) => { const mem = vmMemory(rec, b); + return ((schema.standardHere || {}).boatSizes || []).map(z => z.why ? {...z, note: 'Not on this account', why: 'Not on this account: ' + z.why} + : mem && typeof z.memory_gb === 'number' && z.memory_gb <= mem ? {...z, note: 'Too small', why: `Too small for ${b.label}'s ${mem} GB VM`} : {...z, why: ''}); }; + // The standard's Where step. Machines: hosts (this machine, Settings hosts; one shard each) or Boat sandboxes (a VM + // per unit, as many at once as the account runs), never both. How it runs: on hosts, local processes or a container; + // in a sandbox, or for a benchmark whose units boot VMs of their own (which a container cannot start), fixed. function standardWhere(rec) { const here = schema.standardHere || {}, list = C.whereList(rec), H = schema.hosts || [], b = C.standardBench(rec); + const sandboxes = rec.backend === 'boat', vms = bootsVms(b), noun = unitsOf(rec).noun; + if (vms && rec.backend === 'container') rec.backend = 'local'; // recipe.py refuses it: nothing to choose const backend = rec.backend || 'local'; const selected = C.standardOthers(rec).filter(o => C.chosenOf(rec).includes(o.key)); const dockerWhy = list.includes('local') && here.docker ? 'Not on this machine: ' + here.docker : ''; // Review's rule (nullius/standard/recipe.py): a setup's image installs what has an install; nothing else reaches its container const cannot = selected.some(o => o.setup) ? optionNeeds(rec).host.filter(n => (b.host || {})[n] && !b.host[n].install.length) : []; const containerWhy = dockerWhy || (cannot.length ? cannot.join(', ') + ' cannot be provided in a container: choose local processes.' : ''); - // boat: started where the run runs, with this machine's key (Settings → Sandbox); a host's programs are its prerequisites - const boatWhy = (list.includes('local') ? here.boat : here.boatKey) || ''; const agentRun = b && b.family === 'agent' && selected.some(o => !o.control && !o.setup); + const kind = (v, label, why = '') => ``; const machine = (v, t, sub) => machineTile('std-where', v, t, sub, list.includes(v)); - const how = (v, t, sub, why) => tile({attr: 'std-backend', key: v, on: backend === v && !why, disabled: !!why, why, note: why ? 'Not usable here' : '', mark: v === 'local' ? icon('terminal') : v === 'boat' ? '' : icon('container'), title: t, tip: sub}); + const how = (v, t, sub, why) => tile({attr: 'std-backend', key: v, on: backend === v && !why, disabled: !!why, why, note: why ? 'Not usable here' : v === 'local' ? 'Not sandboxed' : `Each ${noun} in a container`, mark: v === 'local' ? icon('terminal') : icon('container'), title: t, tip: sub}); const dockerOnly = Object.entries((b || {}).environments || {}).filter(([, e]) => !e.local_ok).map(([n]) => n); const web = agentRun && Object.values((b || {}).environments || {}).some(e => e.network); // an agent's public web proxy + // a VM benchmark's provider of one value goes on the fixed line; several are its cards (benchOptions) + const provider = vms && Object.values(b.options || {}).find(o => o.step === 'where' && o.values && o.values.length === 1 && Object.values(o.needs || {}).some(n => n && n.vm)); + // what a sandbox cannot run (recipe.py): a setup or a check runs on a machine; the computer port's socket is on this one + // a benchmark booting VMs of its own runs each unit whole in a VM, with a relay container in each VM (boat_units) + const outside = !sandboxes ? [] : selected.filter(o => !vms && (o.setup || o.control || ((b.needs || []).some(p => p.startsWith('agent.computer/')) && b.family === 'agent'))).map(o => o.label); + let sizes = ''; + if (sandboxes && !here.boat) { + const offered = boatSizes(rec, b), fit = offered.find(z => !z.why); + if (fit && !offered.some(z => z.key === (rec.boat_size || 'small') && !z.why)) rec.boat_size = fit.key; + const size = rec.boat_size || 'small'; + sizes = '

VM size

' + + `
${offered.map(z => tile({attr: 'std-boat-size', key: z.key, on: size === z.key && !z.why, + disabled: !!z.why, why: z.why, note: z.note || z.spec, title: z.label})).join('')}
`; + } return '

Where does it run?

' - + `
${machine('local', 'This machine', 'Nullius runs it here, in runs/' + rec.name + '.')}${H.map(h => machine(h.entry, h.label, 'Over ssh: the standard\'s code at this commit is sent there, uv put there if missing; its job comes back as it runs.')).join('')}
` - + (H.length ? (list.some(w => w !== 'local') ? button('Test connection', 'id="nb-test-host"') + ' ' : '') : `

Hosts are named in ${button('Settings → Hosts', 'data-href="/settings#HOSTS"', 'nb-inlinebutton')}; this machine is the only choice until then.

`) - + (b ? benchOptions(rec, b, 'where') : '') + '

How it runs there

' - + `
${how('local', 'Local processes', 'Its commands run as you, not sandboxed.', '')}` - + how('container', 'Container', web ? 'Sandboxed: each unit in a container whose only ways out are its relay and a public web proxy.' - : 'Sandboxed: each unit in a container whose only way out is its relay.', containerWhy) - + how('boat', 'Boat', 'Sandboxed: each unit in a Boat VM of its own, whose only way out is its relay.', boatWhy) + '
' - + (backend === 'boat' && !boatWhy ? boatSizes(rec) : '') + + `
${kind('hosts', 'Hosts')}${kind('boat', 'Sandboxes', here.boat || '')}
` + + (sandboxes ? `
BoatOne VM per ${esc(noun)}${here.boatSlots ? `${here.boatSlots} at once on this account` : ''}${here.boat ? `${esc(here.boat)}` : ''}
` + sizes + : `
${machine('local', 'This machine', 'Runs here')}${H.map(h => machine(h.entry, h.label, 'Over SSH')).join('')}
` + + (list.some(w => w !== 'local') ? button('Test connection', 'id="nb-test-host"') + ' ' : '')) + + '

Isolation

' + + (sandboxes || vms + ? `
${vms ? `Each ${esc(noun)} in its own VM` : `The agent alone in each ${esc(noun)}'s VM`}` + + `${vms ? (provider ? (sandboxes ? `${esc((provider.names || {})[provider.values[0]] || words(provider.values[0]))}, inside the ${esc(noun)}'s Boat VM` : `${esc(provider.label)}: ${esc((provider.names || {})[provider.values[0]] || words(provider.values[0]))}`) : '') : 'Answers and keys stay on this machine'}
` + : `
${how('local', 'Local processes', 'Its commands run as you, not sandboxed.', '')}` + + how('container', 'Container', web ? 'Sandboxed: each unit in a container whose only ways out are its relay and a public web proxy.' + : 'Sandboxed: each unit in a container whose only way out is its relay.', containerWhy) + '
') + + (b ? benchOptions(rec, b, 'where') : '') + (agentRun && backend === 'local' ? consentBox(rec) : '') - + (dockerOnly.length && backend !== 'container' && agentRun ? `

${esc(dockerOnly.join(', '))}: ${dockerOnly.length > 1 ? 'these environments need' : 'this environment needs'} a container - on ${backend === 'boat' ? 'Boat' : 'local processes'} their units are recorded as not run.

` : '') - + hostSetup(rec); - } - const BOAT_SIZES = [['small', 'Small'], ['default', 'Default'], ['large', 'Large'], ['xlarge', 'XLarge']]; - function boatSizes(rec) { - const here = schema.standardHere || {}, why = here.boatSizes || {}, spec = here.boatSpecs || {}, size = rec.boat_size || 'small'; - const card = ([v, t]) => tile({attr: 'std-boat-size', key: v, on: size === v && !why[v], disabled: !!why[v], - note: why[v] ? 'Not on this account' : spec[v] || '', title: t}); - return '

VM size

' - + `
${BOAT_SIZES.map(card).join('')}
`; + + (outside.length ? `

${esc(outside.join(', '))} cannot run in a Boat VM yet: choose Hosts.

` : '') + + (dockerOnly.length && backend !== 'container' && agentRun ? `

${esc(dockerOnly.join(', '))}: ${dockerOnly.length > 1 ? 'these environments need' : 'this environment needs'} a container - ${sandboxes ? 'in a Boat VM' : 'on local processes'} their units are recorded as not run.

` : '') + + (sandboxes ? '' : hostSetup(rec)); } // What the chosen options of the benchmark need: an embedding model, requirements of the machine (standard.json host). function optionNeeds(rec) { @@ -508,7 +532,7 @@ } // The machines' setup for those requirements: each chosen machine's state, a check, an install on an explicit // click (sudo -n: a machine whose sudo asks for a password is told the commands instead), the commands to copy. - const hostState = {}; // entry -> {missing, error, said} + const hostState = {}; // benchmark:entry:requirement -> {missing, error, said} function hostSetup(rec) { const b = C.standardBench(rec), need = optionNeeds(rec).host; if (!b || !need.length) return ''; @@ -517,31 +541,36 @@ if ((rec.backend || 'local') === 'container' && chosen.length && chosen.every(o => o.setup)) return ''; const reqs = need.map(n => [n, (b.host || {})[n]]).filter(([, r]) => r); const local = ((schema.standardHere || {}).host || {})[b.key] || {}; - // what installs them; none for a machine's own hardware (a GPU), which no package puts there - const lines = reqs.flatMap(([n]) => b.host[n].install).filter((x, i, a) => a.indexOf(x) === i); - const line = w => { - const st = hostState[w] || (w === 'local' ? {missing: Object.fromEntries(need.filter(n => local[n]).map(n => [n, local[n]]))} : null); - const gone = st && st.missing ? Object.entries(st.missing).filter(([n]) => need.includes(n)) : []; - const state = !st ? 'not checked' : st.error ? `${esc(st.error)}` - : gone.length ? `${esc(gone.map(([, ps]) => ps.join(', ')).join(', '))}` : 'ready ✓'; - return `
  • ${esc(w === 'local' ? 'This machine' : C.machineLabel(w))} ${state} ${button('Check', `data-hostreq-check="${esc(w)}"`, 'nb-inlinebutton')}${lines.length && (gone.length || (st && st.error)) ? ' ' + button('Install for me', `data-hostreq-install="${esc(w)}"`, 'nb-inlinebutton') : ''}${st && st.said ? ` ${esc(st.said)}` : ''}
  • `; - }; - return `

    Machine setup

    Needed by the options you chose, on each machine it runs on
    ` - + reqs.map(([n, r]) => `

    ${esc(r.label)}: ${esc(r.summary)}

    `).join('') - + `
      ${C.whereList(rec).map(line).join('')}
    ` - + (lines.length ? `
    or run it yourself, on each machine
    ${esc(lines.join('\n'))}
    ` : ''); + return reqs.map(([name, req]) => { + const lines = req.install || []; + const machines = C.whereList(rec); + const rows = machines.map(w => { + const st = hostState[b.key + ':' + w + ':' + name] || (w === 'local' ? {missing: local[name] ? {[name]: local[name]} : {}} : null); + const missing = st && st.missing && st.missing[name]; + const state = !st ? 'Not checked' : st.error ? st.error : missing && missing.length ? 'Missing' : 'Ready'; + const attrs = `data-hostreq-name="${esc(name)}"`; + return `
    ${machines.length > 1 || w !== 'local' ? `${esc(w === 'local' ? 'This machine' : C.machineLabel(w))}` : ''}` + + `${esc(req.summary || req.label)}` + + `
    ${esc(state)}` + + button('Check', `data-hostreq-check="${esc(w)}" ${attrs}`, 'nb-button') + + (lines.length && (missing && missing.length || st && st.error) ? button('Install', `data-hostreq-install="${esc(w)}" ${attrs}`, 'nb-button nb-primary') : '') + + `
    ${st && st.said ? `${esc(st.said)}` : ''}
    `; + }).join(''); + return `

    ${esc(req.label)}

    ${rows}` + + (lines.length ? `
    Install manually
    ${esc(lines.join('\n'))}
    ` : '') + '
    '; + }).join(''); } - async function hostReq(entry, what) { - const rec = r(), b = C.standardBench(rec); - hostState[entry] = {...(hostState[entry] || {}), said: what === 'install' ? 'installing…' : 'checking…', error: ''}; + async function hostReq(entry, what, name) { + const rec = r(), b = C.standardBench(rec), key = b.key + ':' + entry + ':' + name; + hostState[key] = {...(hostState[key] || {}), said: what === 'install' ? 'Installing…' : 'Checking…', error: ''}; renderForm(); try { const res = await nulliusFetch('/new-run/host/' + what, {method: 'POST', headers: {'X-CSRF-Token': schema.csrf}, - body: new URLSearchParams({entry, benchmark: b.key, requirements: optionNeeds(rec).host.join(',')})}); + body: new URLSearchParams({entry, benchmark: b.key, requirements: name})}); const j = await res.json(); - if (what === 'install' && j.ok) return hostReq(entry, 'check'); - hostState[entry] = j.ok ? {missing: j.missing || {}} : {...(hostState[entry] || {}), error: j.error || ('HTTP ' + res.status), said: j.manual ? 'run the commands below by hand' : ''}; - } catch (e) { hostState[entry] = {error: String(e)}; } + if (what === 'install' && j.ok) return hostReq(entry, 'check', name); + hostState[key] = j.ok ? {missing: j.missing || {}} : {...(hostState[key] || {}), error: j.error || ('HTTP ' + res.status), said: j.manual ? 'Use Install manually below' : ''}; + } catch (e) { hostState[key] = {error: String(e)}; } renderForm(); } // A benchmark's own options (a search, a mode), for the chosen datasets they apply to, and what a value needs of this @@ -565,6 +594,7 @@ }) .map(([n, o]) => { const cur = (rec.bench_options || {})[n] ?? o.default; + if (scope === 'where' && o.values.length === 1 && Object.values(o.needs || {}).some(x => x && x.vm)) return ''; // on the How it runs line if (scope === 'where' && o.values.length === 1) return `

    ${esc(o.label)}

    ${esc((o.names || {})[cur] || cur)}

    `; if (o.pattern) return ``; const tileOf = v => { const need = (o.needs || {})[v] || {}; @@ -727,7 +757,7 @@ const ds = sets.length === 1 ? sets[0] : null, all = ds && ds.units ? String(ds.units) : 'all'; return '

    Scope

    ' + field('Number of ' + u.nouns, 'limit', rec.limit, {type: 'number', min: 1, placeholder: all, hint: 'The first ones' + per + ', in its order.'}) - + field('Concurrent ' + u.nouns, 'concurrency', rec.concurrency, {type: 'number', min: 1, max: 32, placeholder: '1'}) + '
    ' + + field('Concurrent ' + u.nouns, 'concurrency', rec.concurrency, {type: 'number', min: 1, max: 32, placeholder: rec.backend === 'boat' ? String((schema.standardHere || {}).boatSlots || 'Boat\'s limit') : '1'}) + '' + '
    ' + field('Attempts per ' + u.noun, 'repeat', rec.repeat || '', {type: 'number', min: 1, max: 32, placeholder: '1'}) + field('Retries per attempt', 'retries', rec.retries ?? '', {type: 'number', min: 0, max: 9, placeholder: '0'}) + '
    ' // a conversation's most agent turns (recipe.py: run --max-turns): only a benchmark that holds conversations has turns @@ -970,7 +1000,7 @@ root.addEventListener('submit',event=>{if(event.target.id==='nb-save-form'){event.preventDefault();save();}}); // Running from here: the runner script is posted with the target; the page follows the log until the run ends. const runPanel=()=>{ - const rec=r(), where=C.whereList(rec).map(w=>w==='local'?'this machine':C.machineLabel(w)).join(' + '); + const rec=r(), where=C.isStandard(rec)&&rec.backend==='boat'?'Boat sandboxes':C.whereList(rec).map(w=>w==='local'?'this machine':C.machineLabel(w)).join(' + '); if(!schema.local)return note('Run from the application host','Runs start from a browser on the machine Nullius runs on.'); // A provider not set in Settings: Run waits and says why. const blocked=C.providerBlocker(rec); @@ -1051,8 +1081,8 @@ // key, the keyboard goes back to that field if(d.servedRemove!==undefined){event.preventDefault();C.removeServed(r(),d.servedRole,d.servedRemove);syncServed();refreshPlan();if(event.detail===0)root.querySelector(`input[data-served-by="${d.servedRole}"]`)?.focus();return;} if(d.benchOpt!==undefined){const rec=r(),o=((C.standardBench(rec)||{}).options||{})[d.benchOpt]||{};rec.bench_options={...(rec.bench_options||{})};if(d.benchVal===o.default)delete rec.bench_options[d.benchOpt];else rec.bench_options[d.benchOpt]=d.benchVal;renderForm();refreshPlan();return;} - if(d.hostreqCheck!==undefined){hostReq(d.hostreqCheck,'check');return;} - if(d.hostreqInstall!==undefined){hostReq(d.hostreqInstall,'install');return;} + if(d.hostreqCheck!==undefined){hostReq(d.hostreqCheck,'check',d.hostreqName);return;} + if(d.hostreqInstall!==undefined){hostReq(d.hostreqInstall,'install',d.hostreqName);return;} if(d.copyCode!==undefined){const t=b.closest('.std-code').querySelector('code').textContent;try{await navigator.clipboard.writeText(t);b.textContent='Copied';}catch{b.textContent='Select it';}setTimeout(()=>{b.textContent='Copy';},1500);return;} if(d.pickCat!==undefined){pickOf().cat=d.pickCat;applyPick();return;} if(d.pickFam!==undefined){pickOf().fam=d.pickFam;applyPick();return;} @@ -1072,6 +1102,8 @@ else if(d.stdOther){const rec=r(),c=C.chosenOf(rec);rec.others=c.includes(d.stdOther)?c.filter(k=>k!==d.stdOther):[...c,d.stdOther];delete rec.other;if(rec.options)delete rec.options[d.stdOther];} // the standard's machines: each a toggle, one at least; several split the run into shards else if(d.stdWhere){const rec=r(),list=C.whereList(rec),on=list.includes(d.stdWhere),next=on?list.filter(w=>w!==d.stdWhere):[...list,d.stdWhere];if(!next.length)return;rec.where=next.length>1?next:next[0];} + // hosts or Boat sandboxes: a sandbox run starts from this machine alone + else if(d.stdMachines){const rec=r();if(d.stdMachines==='boat'){rec.backend='boat';rec.where='local';}else if(rec.backend==='boat')rec.backend='local';} else if(d.stdBackend){r().backend=d.stdBackend;} else if(d.stdBoatSize){r().boat_size=d.stdBoatSize;} else if(d.harborSource){r().source=d.harborSource;} diff --git a/app/tests/catalogue_docs.py b/app/tests/catalogue_docs.py index c087d490..7bef9284 100644 --- a/app/tests/catalogue_docs.py +++ b/app/tests/catalogue_docs.py @@ -231,7 +231,8 @@ def _values(spec: dict) -> str: def _needs_rows(name: str, spec: dict) -> list: - return [[f"`{name}`", f"`{val}`", "; ".join(f"{k}: {', '.join(w) if isinstance(w, list) else w}" for k, w in need.items())] + said = lambda k, w: f"a {w['memory_gb']:g} GB VM per unit" if k == "vm" else ", ".join(w) if isinstance(w, list) else w + return [[f"`{name}`", f"`{val}`", "; ".join(f"{k}: {said(k, w)}" for k, w in need.items())] for val, need in (spec.get("needs") or {}).items()] diff --git a/app/tests/test_standard_bench_options.py b/app/tests/test_standard_bench_options.py index 94e4dfa4..c66b46c7 100644 --- a/app/tests/test_standard_bench_options.py +++ b/app/tests/test_standard_bench_options.py @@ -38,6 +38,14 @@ def test_a_value_may_only_need_a_declared_requirement_and_a_known_dataset(self): problems = " ".join(definitions.problems({k: v for k, v in bad.items() if not k.startswith("_")}, "benchmark")) for said in ("options.retrieval.datasets", "needs.alltools.host", "needs.alltools.embeddings"): self.assertIn(said, problems) + # a unit's own VM is a Where option's, and says its memory in GB alone + for o, vm in ((TAU["options"]["retrieval"], {"memory_gb": 4}), (dict(TAU["options"]["retrieval"], step="where"), {"memory_gb": "4G"}), + (dict(TAU["options"]["retrieval"], step="where"), {"memory_gb": 4, "cpus": 4})): + bad = tau(options={"retrieval": dict(o, needs={"bm25": {"vm": vm}})}) + self.assertIn("needs.bm25.vm", " ".join(definitions.problems({k: v for k, v in bad.items() if not k.startswith("_")}, "benchmark"))) + good = tau(options={"retrieval": dict(TAU["options"]["retrieval"], step="where", needs={"bm25": {"vm": {"memory_gb": 4}}})}) + self.assertNotIn("needs.bm25", " ".join(definitions.problems({k: v for k, v in good.items() if not k.startswith("_")}, "benchmark"))) + self.assertEqual((definitions.vm_memory_gb(good, {"retrieval": "bm25"}), definitions.vm_memory_gb(good, {"retrieval": "terminal_use"})), (4, 0)) worse = tau(host={"sandbox": {"summary": "x", "check": ["srt; rm"], "install": {"brew": ["x"]}}}) problems = " ".join(definitions.problems({k: v for k, v in worse.items() if not k.startswith("_")}, "benchmark")) self.assertIn("host.sandbox.check", problems) diff --git a/app/tests/test_standard_boat.py b/app/tests/test_standard_boat.py index 54fcba34..404ff0fd 100644 --- a/app/tests/test_standard_boat.py +++ b/app/tests/test_standard_boat.py @@ -119,6 +119,22 @@ def test_a_run_longer_than_the_accounts_starts_is_said(self): def test_slots_is_the_accounts_limit_of_vms_at_once(self): self.assertEqual(boat.slots(boat.Client("k", opener=FakeAPI([]))), 2) + def test_one_answer_of_its_limits_says_both_its_sizes_and_its_vms_at_once(self): + api = FakeAPI([]) + client = boat.Client("k", opener=api) + limits = client.limits() + self.assertEqual((boat.slots(client, limits), boat.machine_types(client, limits)["large"]), (2, "not during the free trial")) + self.assertEqual(len(api.calls), 1) + self.assertIsNone(boat.slots(client, {"maxActiveSandboxes": True})) + + def test_a_size_fits_a_units_own_vm_only_with_more_memory_than_it(self): + # measured: OSWorld's 4 GB guest is refused on small (4 GB, 2 free) and boots on default (8 GB) + self.assertFalse(boat.fits("small", 4)) + self.assertTrue(boat.fits("default", 4)) + self.assertTrue(boat.fits("small", 0)) + self.assertTrue(boat.fits("large", 4)) # its memory unmeasured: not refused here + self.assertEqual((boat.spec("small"), boat.spec("large")), ("2 vCPU, 4 GB", "")) + def test_sweep_deletes_only_this_jobs_vms(self): api = FakeAPI([]) self.assertEqual(boat.sweep("job.1", boat.Client("k", opener=api)), 1) @@ -514,11 +530,12 @@ def test_the_runs_size_is_every_vms_and_one_the_account_cannot_start_is_refused( with self.assertRaisesRegex(run.Refused, "Boat cannot start a large VM: not during the free trial"): run.Run(b, a, model="m", provider="custom", env=self.env, jobs_root=self.tmp, backend="boat", boat_size="large") - def test_a_check_runs_here_and_no_key_means_no_run(self): + def test_a_check_is_refused_not_run_here_and_no_key_means_no_run(self): b = definitions.get("benchmark", "fakebench", ROOT) b["controls"] = {"nop": {"source": "nullius", "from": "nothing said"}} nop = definitions.control(b, "nop") - self.assertEqual(run.Run(b, nop, model="m", provider="custom", env=self.env, jobs_root=self.tmp, backend="boat").backend, "local") + with self.assertRaisesRegex(run.Refused, "runs on a machine, not in a Boat VM"): + run.Run(b, nop, model="m", provider="custom", env=self.env, jobs_root=self.tmp, backend="boat") a = definitions.get("agent", "fake", ROOT) with mock.patch.object(boat, "unusable", lambda env=None, which=None: "no Boat API key: set it in Settings → Sandbox"): with self.assertRaisesRegex(run.Refused, "Boat backend cannot run here: no Boat API key"): @@ -539,16 +556,22 @@ def test_boat_needs_no_yes_and_runs_here_or_on_a_host(self): self.assertFalse(out["docker"]) sized = recipe.compile(a_recipe(backend="boat", boat_size="default"), defs(), boat_unusable=lambda here=True: None) self.assertIn("--boat-size default", sized["script"]) - self.assertIn("default VMs", sized["review"][-1][2]) + self.assertIn("Boat sandboxes · one Default VM per ", sized["review"][-1][2]) self.assertEqual([e["step"] for e in recipe.compile(a_recipe(backend="boat", boat_size="huge"), defs())["errors"]], [recipe.STEP["where"]]) - self.assertIn("boat backend", out["review"][-1][2]) - # on a host: the key is this machine's, the programs the host's (its prerequisites) + # as many units at once as the account runs, unless the recipe says fewer; the run caps it (boat.slots) + every = recipe.compile(a_recipe(backend="boat", concurrency=""), defs(), boat_unusable=lambda here=True: None) + self.assertIn(f"-j {recipe.MAX_CONCURRENT} ", every["script"]) + self.assertIn("as many at a time as Boat allows", next(r[2] for r in every["review"] if r[1] == "Scope")) + fewer = recipe.compile(a_recipe(backend="boat", concurrency=3), defs(), boat_unusable=lambda here=True: None) + self.assertIn("-j 3 ", fewer["script"]) + + def test_hosts_and_sandboxes_do_not_mix(self): asked = [] - out = recipe.compile(a_recipe(backend="boat", where="me@bench"), defs(), boat_unusable=lambda here=True: asked.append(here)) - self.assertEqual(out["errors"], []) - self.assertEqual(asked, [False]) - self.assertIn("--backend boat", out["script"]) + for where in ("me@bench", ["local", "me@bench"]): + out = recipe.compile(a_recipe(backend="boat", where=where), defs(), boat_unusable=lambda here=True: asked.append(here)) + self.assertEqual([e["message"] for e in out["errors"]], ["Hosts and Boat sandboxes do not mix: choose Hosts, or Sandboxes alone."]) + self.assertEqual(asked, []) def test_a_size_the_account_cannot_start_is_the_where_steps_error(self): problem = lambda size: "not during the free trial" if size == "large" else None @@ -558,6 +581,23 @@ def test_a_size_the_account_cannot_start_is_the_where_steps_error(self): out = recipe.compile(a_recipe(backend="boat", boat_size="default"), defs(), boat_unusable=lambda here=True: None, boat_size_problem=problem) self.assertEqual(out["errors"], []) + def test_a_benchmark_booting_its_own_vms_wants_a_size_that_holds_them_and_never_a_container(self): + # OSWorld's Original setup, on the bundled catalogue: the size is judged from its definition's VM, not its name + base = {"harness": recipe.HARNESS, "name": "os", "benchmark": "osworld2", "datasets": ["offline36"], "others": ["original"], + "model": "MiniMax-M3", "provider": "openrouter", "where": "local"} + errors = lambda **k: [e["message"] for e in recipe.compile(dict(base, **k), boat_unusable=lambda here=True: None)["errors"] + if e["step"] == recipe.STEP["where"]] + small = errors(backend="boat", boat_size="small") + self.assertIn("A Small VM is too small for OSWorld2, whose units boot a 4 GB VM: choose a larger size.", small) + # Model-calling setups and controls use a relay container inside each whole-task VM. + self.assertNotIn("Original cannot run in a Boat VM yet: choose Hosts.", small) + self.assertEqual(errors(backend="boat", boat_size="default", others=["original", "nop"]), []) + self.assertFalse([m for m in errors(backend="boat", boat_size="default") if "too small" in m]) + self.assertIn("OSWorld2 starts each unit's VM on the machine: choose local processes.", errors(backend="container")) + self.assertFalse([m for m in errors(backend="local") if "VM" in m]) + self.assertTrue(definitions.boots_vms(definitions.get("benchmark", "osworld1"))) + self.assertFalse(definitions.boots_vms(definitions.get("benchmark", "gsm8k"))) + def test_without_its_key_the_where_step_says_so(self): out = recipe.compile(a_recipe(backend="boat"), defs(), boat_unusable=lambda here=True: "no Boat API key: set it in Settings → Sandbox") self.assertEqual([e["step"] for e in out["errors"]], [recipe.STEP["where"]]) diff --git a/app/tests/test_standard_boat_units.py b/app/tests/test_standard_boat_units.py new file mode 100644 index 00000000..6c862248 --- /dev/null +++ b/app/tests/test_standard_boat_units.py @@ -0,0 +1,125 @@ +"""A benchmark whose units boot their own VM, on Boat: each unit whole in a VM of its own, as a shard of one.""" +from __future__ import annotations + +import shlex +import io +import json +import os +import subprocess +import tempfile +from types import SimpleNamespace +from unittest.mock import patch +import unittest + +from nullius import secrets +from nullius.standard import boat_units, cli, definitions, recipe, run, container + +OSWORLD1 = definitions.get("benchmark", "osworld1") + + +class Plan(unittest.TestCase): + def test_each_shard_is_one_unit_and_a_limit_runs_the_first_ones(self): + self.assertEqual(boat_units.plan(OSWORLD1, "test_small", None), (39, list(range(1, 40)))) + self.assertEqual(boat_units.plan(OSWORLD1, "test_small", 2), (39, [1, 2])) + self.assertEqual(boat_units.plan(OSWORLD1, "test_small", 500)[1][-1], 39) + with self.assertRaisesRegex(boat_units.Refused, "no dataset"): + boat_units.plan(OSWORLD1, "nowhere", None) + bare = dict(OSWORLD1, datasets=[{"id": "x"}]) + with self.assertRaisesRegex(boat_units.Refused, "declares no number of units"): + boat_units.plan(bare, "x", None) + + def test_the_vm_runs_this_runners_snapshot_on_one_unit_on_local_processes(self): + line = boat_units.child_argv(["osworld1", "nop", "--dataset", "test_small"], app="/home/user/.nullius/app/v1-x", + index=3, n=39, launch="L", name="job__shard-3of39", jobs="/j") + argv = shlex.split(line.split(" && ", 1)[1]) + self.assertEqual(argv[:4], ["python3", "-I", "-S", "-B"]) + self.assertEqual(argv[argv.index("--shard") + 1], "3/39") + self.assertEqual(argv[argv.index("--shard-by") + 1], "position") + self.assertIn("--relay-container", argv) + self.assertIn("--unsandboxed", argv) + self.assertNotIn("--backend", argv) # local processes: the VM is the sandbox + self.assertIn("NULLIUS_CATALOGUE=/home/user/.nullius/app/v1-x/catalogue", line) + + +class Compile(unittest.TestCase): + def test_on_boat_a_vm_benchmark_runs_its_null_control_whole_in_each_vm_and_asks_this_machine_nothing(self): + base = {"harness": recipe.HARNESS, "name": "os", "benchmark": "osworld1", "datasets": ["test_small"], + "others": ["nop"], "provider": "openrouter", "where": "local", "backend": "boat", "boat_size": "default", "limit": 2} + out = recipe.compile(base, boat_unusable=lambda here=True: None) + self.assertEqual(out["errors"], []) + self.assertIn("--backend boat --boat-size default", out["script"]) + self.assertEqual(out["host"]["needs"], []) # each VM installs what the options need + self.assertFalse(out["docker"]) + # Model calls use a separate relay container in the Boat VM. + agent = recipe.compile(dict(base, others=["original"], model="gpt-4o"), boat_unusable=lambda here=True: None) + self.assertEqual(agent["errors"], []) + + + + +class RelayPlacement(unittest.TestCase): + def test_local_task_uses_container_relay_with_host_url_and_judge_route(self): + with tempfile.TemporaryDirectory() as d: + r = run.Run.__new__(run.Run) + r.base, r.key, r.backend, r.relay_container = "https://example.test/v1", "fake-secret", "local", True + r.other, r.key_header = {}, "authorization" + r.judge_model, r.simulator_model = "grader", None + r.name, r.image, r.model = "job", "image", "agent-model" + r.prices, r.serving, r.max_calls, r.call_seconds = {}, {}, 3, 12 + r.ledger_file = lambda: "/ledger" + r.models_called = lambda: ["agent-model", "grader"] + with patch.object(container, "Unit") as unit, patch.object(run.relay, "Relay") as local: + unit.return_value.start.return_value = unit.return_value + unit.return_value.url.return_value = "http://127.0.0.1:1234/u0001/v1" + got = r.unit_relay("u0001", d, {}) + local.assert_not_called() + self.assertEqual(got.unit_url, "http://127.0.0.1:1234/u0001/v1") + args = unit.call_args.kwargs + self.assertEqual(args["key"], "fake-secret") + self.assertEqual(args["routes"], {"u0001-judge": "grader"}) + self.assertEqual((args["ledger"], args["max_calls"]), ("/ledger", 3)) + + def test_bootstrap_gets_credentials_from_stdin_not_arguments(self): + with tempfile.TemporaryDirectory() as d: + os.makedirs(d + "/nullius/standard") + with open(d + "/nullius/standard/cli.py", "w") as f: + f.write('import os; print(os.environ["OPENAI_API_KEY"])') + result = subprocess.run(["python3", "-I", "-S", "-B", "-c", boat_units.RUNPY, d], + input=json.dumps({"OPENAI_API_KEY": "fake-secret"}) + "\n", + text=True, capture_output=True, check=True) + self.assertEqual(result.stdout.strip(), "fake-secret") + broken = subprocess.run(["python3", "-I", "-S", "-B", "-c", boat_units.RUNPY, d], + input="", text=True, capture_output=True) + self.assertNotEqual(broken.returncode, 0) + self.assertEqual(broken.stdout, "") + + def test_credentials_include_only_selected_provider_and_declared_licences(self): + args = SimpleNamespace(provider="openrouter", from_run=True) + held = SimpleNamespace(kinds={"HF_TOKEN": "token"}, get=lambda name: {"kind": "token", "value": "fake-hf-token"}) + with patch.dict(os.environ, {"OPENAI_API_KEY": "fake-model-key", "BOAT_API_KEY": "fake-boat-key", + "UNRELATED_KEY": "unrelated"}, clear=True), \ + patch.object(cli, "licences", return_value=(held, None)): + got = cli.boat_credentials(args, OSWORLD1, {"_kind": "upstream"}, {}) + self.assertEqual(set(got), {"OPENAI_API_KEY", "OPENAI_BASE_URL", secrets.HANDED}) + self.assertEqual(got["OPENAI_API_KEY"], "fake-model-key") + self.assertEqual(secrets.unpack(got[secrets.HANDED]), {"HF_TOKEN": b"fake-hf-token"}) + + def test_cli_forwards_models_options_and_divides_cap(self): + with patch.object(boat_units, "run", return_value=("/job", False)) as launch, \ + patch.object(cli, "boat_credentials", return_value={"OPENAI_API_KEY": "fake-secret"}), \ + patch("sys.stdout", new_callable=io.StringIO): + result = cli.main(["run", "osworld1", "original", "--backend", "boat", "--boat-size", "default", + "--model", "gpt-4o", "--provider", "openrouter", "--dataset", "test_small", + "--limit", "2", "--max-usd", "1", "--max-calls", "5", "--served-by", "openai", + "--price", "openai/gpt-4o=1,2", "--timeout", "agent=90"]) + self.assertEqual(result, 0) + words = launch.call_args.args[1] + for flag, value in (("--model", "gpt-4o"), ("--max-usd", "0.5"), ("--max-calls", "5"), + ("--served-by", "openai"), ("--price", "openai/gpt-4o=1,2"), ("--timeout", "agent=90")): + self.assertEqual(words[words.index(flag) + 1], value) + self.assertNotIn("fake-secret", " ".join(words)) + self.assertEqual(launch.call_args.kwargs["credentials"], {"OPENAI_API_KEY": "fake-secret"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/app/tests/test_standard_run.py b/app/tests/test_standard_run.py index 096b43bc..f9e60aef 100644 --- a/app/tests/test_standard_run.py +++ b/app/tests/test_standard_run.py @@ -330,6 +330,19 @@ def test_a_term_runs_the_cleanup_and_a_second_term_cannot_cut_it(self): self.assertEqual(p.returncode, 143) self.assertTrue(os.path.exists(done)) + def test_local_agent_calls_a_separate_container_relay(self): + from tests import docker_tier + why = docker_tier.why_not() + if why: + self.skipTest(why) + directory = self.go("fakebench", "fake", relay_container=True) + got, _ = self.outcomes(directory) + self.assertEqual(got["good"].outcome, classify.RESOLVED) + self.assertEqual(got["model"].outcome, classify.FAILED_VERIFICATION) + self.assertTrue(self.relay_calls(directory)) + self.assert_no_key(directory) + self.assert_unit_relays(directory) + def test_an_agent_run_reads_back_with_its_outcomes(self): job_dir = self.go("fakebench", "fake") got, j = self.outcomes(job_dir) diff --git a/app/tests/test_standard_shard.py b/app/tests/test_standard_shard.py index 1762cfad..37a29399 100644 --- a/app/tests/test_standard_shard.py +++ b/app/tests/test_standard_shard.py @@ -32,6 +32,12 @@ def read(path): class Partition(unittest.TestCase): UNITS = [f"set/{i}" for i in range(500)] + def test_by_position_shard_i_of_n_holds_the_units_at_places_i_minus_1_mod_n(self): + units = ["a", "b", "c", "d", "e"] + self.assertEqual([shard.select(units, i, 5, by="position") for i in (1, 5)], [[(0, "a")], [(4, "e")]]) + self.assertEqual(shard.select(units, 2, 2, by="position"), [(1, "b"), (3, "d")]) + self.assertEqual(shard.record(1, 5, 5, ["a"], by="position")["partition"], shard.BY["position"]) + def test_every_unit_is_in_exactly_one_shard(self): for n in (1, 2, 3, 7): got = [u for i in range(1, n + 1) for _, u in shard.select(self.UNITS, i, n)] diff --git a/app/ui_tests/test_agents.py b/app/ui_tests/test_agents.py index e3a21583..7dd77445 100644 --- a/app/ui_tests/test_agents.py +++ b/app/ui_tests/test_agents.py @@ -523,9 +523,23 @@ def test_the_builder_is_given_the_controls_their_proofs_and_whether_docker_runs_ self.assertEqual([(c["key"], c["source"], c["proven"], c["failed"]) for c in ds["controls"]], [("nop", "nullius", None, None), ("oracle", "adapter", None, None)]) # no benchmark here needs a machine setup + sizes = [{"key": "small", "label": "Small", "spec": "2 vCPU, 4 GB", "memory_gb": 4, "why": None}, + {"key": "default", "label": "Default", "spec": "4 vCPU, 8 GB", "memory_gb": 8, "why": None}, + {"key": "large", "label": "Large", "spec": "", "memory_gb": None, "why": None}, + {"key": "xlarge", "label": "XLarge", "spec": "", "memory_gb": None, "why": None}] self.assertEqual(schema["standardHere"], {"docker": None, "boat": "no Boat API key: set it in Settings → Sandbox", - "boatKey": "no Boat API key: set it in Settings → Sandbox", "boatSizes": {}, "boatSpecs": {"small": "2 vCPU, 4 GB", "default": "4 vCPU, 8 GB"}, - "licences": {}, "host": {}}) + "boatSizes": sizes, "boatSlots": None, "licences": {}, "host": {}}) + + def test_the_builder_offers_the_boat_sizes_and_vms_at_once_its_account_says(self): + from nullius.standard import boat + self.fixture() + account = {"types": {"small": None, "default": None, "large": "not during the free trial"}, "slots": 2} + with patch.object(boat, "unusable", lambda here=True: None), patch.object(web.execute, "boat_account", lambda: account): + html = self.get("/new-run").get_data(as_text=True) + here = json.loads(re.search(r'', html, re.S).group(1))["standardHere"] + self.assertEqual(here["boatSlots"], 2) + self.assertEqual([(z["key"], z["why"]) for z in here["boatSizes"]], + [("small", None), ("default", None), ("large", "not during the free trial")]) # xlarge: not the account's def test_docker_is_refused_on_the_where_step_with_this_machines_reason(self): self.fixture() diff --git a/app/ui_tests/test_builder_where.py b/app/ui_tests/test_builder_where.py index 28dac581..478b4fb5 100644 --- a/app/ui_tests/test_builder_where.py +++ b/app/ui_tests/test_builder_where.py @@ -8,42 +8,100 @@ class StandardWhere(CompilerCase): - def draw(self, docker, edit=""): + SIZES = [{"key": "small", "label": "Small", "spec": "2 vCPU, 4 GB", "memory_gb": 4, "why": None}, + {"key": "default", "label": "Default", "spec": "4 vCPU, 8 GB", "memory_gb": 8, "why": None}, + {"key": "large", "label": "Large", "spec": "", "memory_gb": None, "why": "not during the free trial"}] + + def draw(self, docker, edit="", backends=("local", "docker"), boat=None, slots=2, host=None): """standardWhere and what it calls, cut from the page script, run against the core: {benchmark backend: html}; - ``edit``, JavaScript changing the recipe ``r`` first.""" + ``edit``, JavaScript changing the recipe ``r`` first; ``boat``, why Boat cannot run here.""" page = (APP / "static/builder/command-builder.js").read_text(encoding="utf-8") cut = lambda start, end: page[page.index(start):page.index(end, page.index(start))] line = lambda name: re.search(r"^ const %s = .*$" % name, page, re.M).group(0) + "\n" - schema = dict(self.schema, hosts=[], standardHere={"docker": docker, "licences": {}, "host": {}}) + schema = dict(self.schema, hosts=[], standardHere={"docker": docker, "licences": {}, "host": host or {}, "boat": boat, + "boatSizes": self.SIZES, "boatSlots": slots}) program = ("const schema=" + json.dumps(schema) + ";const C=require(" + json.dumps(str(APP / "static/builder/command-builder-core.js")) + ")(schema);\n" - + line("esc") + line("button") + cut(" const tile = t =>", " const picker = ") + + line("esc") + line("button") + cut(" const unitsOf = ", " // the first clause") + line("words") + + cut(" const tile = t =>", " const picker = ") + cut(" const consentBox = ", " async function hostReq(") + cut(" function benchOptions(", " // Only credentials") - + "const out={};for(const b of C.standard().benchmarks)for(const backend of ['local','docker']){" + + "const out={};for(const b of C.standard().benchmarks)for(const backend of " + json.dumps(list(backends)) + "){" + "const r=C.createStandard(b.key);r.backend=backend;" + edit + "try{out[b.key+' '+backend]=standardWhere(r);}catch(e){out[b.key+' '+backend]=String(e);}}" + "process.stdout.write(JSON.stringify(out));") return json.loads(subprocess.run([self.node, "-"], input=program, check=True, capture_output=True, text=True).stdout) + def test_hosts_or_sandboxes_never_both_and_a_sandbox_is_its_own_isolation(self): + hosts, boxes = self.draw(None), self.draw(None, backends=("boat",)) + card = lambda html, v: re.search(r'", html) + self.assertIn("2 at once on this account", html) # the account's VMs at once, from its API + self.assertNotIn("data-std-where", html) # no host to add to a sandbox run + self.assertNotIn("data-std-backend", html) # nothing to choose: the VM is the sandbox + self.assertNotIn("not sandboxed", html.lower()) + self.assertIn('class="nb-fixed std-isolation"', html) + self.assertIn('data-std-boat-size="large" aria-pressed="false" disabled aria-disabled="true" title="Not on this account: not during the free trial"', html) + for name, html in hosts.items(): + with self.subTest(name): + self.assertIn('aria-pressed="true"', card(html, "hosts")) + self.assertIn('data-std-where="local"', html) + self.assertNotIn("data-std-boat-size", html) + ordinary = boxes["humaneval boat"] + self.assertIn("The agent alone in each problem's VM", ordinary) + self.assertIn("Answers and keys stay on this machine", ordinary) + self.assertIn('class="std-boat-card"', ordinary) + self.assertNotIn("shards, merged", ordinary) + self.assertNotIn("at once", self.draw(None, backends=("boat",), slots=None)["gsm8k boat"]) # the API said nothing: no number + gone = self.draw(None, backends=("boat",), boat="no Boat API key: set it in Settings → Sandbox")["gsm8k boat"] + self.assertIn('disabled aria-disabled="true" title="no Boat API key: set it in Settings → Sandbox"', card(gone, "boat")) + self.assertNotIn("data-std-boat-size", gone) + + def test_a_benchmark_booting_its_own_vms_asks_no_isolation_and_refuses_a_size_too_small_for_its_vm(self): + for key in ("osworld1", "osworld2"): + with self.subTest(key): + hosts = self.draw(None)[key + " local"] + self.assertNotIn("data-std-backend", hosts) + self.assertIn('Each task in its own VMVM provider: Docker', hosts) + self.assertEqual(hosts.count("VM provider"), 1) + boxes = self.draw(None, backends=("boat",))[key + " boat"] + label = "OSWorld1" if key == "osworld1" else "OSWorld2" + self.assertIn(f'data-std-boat-size="small" aria-pressed="false" disabled aria-disabled="true" title="Too small for {label}'s 4 GB VM"', boxes) + self.assertIn('data-std-boat-size="default" aria-pressed="true"', boxes) # the smallest that fits, chosen for it + self.assertIn("Each task in its own VMDocker, inside the task's Boat VM", boxes) + self.assertNotIn("cannot run in a Boat VM", boxes) + # Null controls run whole in each VM, including OSWorld2's grader. + nop = self.draw(None, "r.others=['nop'];", backends=("boat",)) + self.assertNotIn("cannot run in a Boat VM", nop["osworld1 boat"]) + self.assertNotIn("cannot run in a Boat VM", nop["osworld2 boat"]) + # an agent's run still asks the yes to running as you: its process is on the machine, only the desktop in a VM + agent = self.draw(None, "r.others=[C.standardOthers(r).find(o=>!o.setup&&!o.control&&!o.why.length).key];")["osworld1 local"] + self.assertIn("data-std-unsandboxed", agent) + # a saved container recipe is brought back to the machine: a container cannot start the VMs + self.assertNotIn("data-std-backend", self.draw(None)["osworld2 docker"]) + def test_every_standard_benchmark_draws_its_where_step_and_says_why_docker_is_not_offered_once(self): missing, here = self.draw("docker is not installed"), self.draw(None) self.assertEqual(len(missing), 2 * len(self.schema["standard"]["benchmarks"])) for name, html in missing.items(): with self.subTest(name): self.assertTrue(html.startswith("]*data-std-backend="container"[^>]*>', html).group(0) self.assertIn('title="Not on this machine: docker is not installed"', tile) self.assertEqual(html.count("Not on this machine"), 1) - container = re.search(r'', html) + self.assertIn('data-hostreq-name="scoring_programs"', html) + self.assertIn('sudo apt-get install', html) def test_an_option_shows_its_example_and_a_setup_its_label(self): page = (APP / "static/builder/command-builder.js").read_text(encoding="utf-8") diff --git a/app/ui_tests/test_catalogue_routes.py b/app/ui_tests/test_catalogue_routes.py index 9b3de520..7e8430cf 100644 --- a/app/ui_tests/test_catalogue_routes.py +++ b/app/ui_tests/test_catalogue_routes.py @@ -249,7 +249,7 @@ def test_a_machine_setup_says_what_checks_it_and_what_installs_it(self): def test_a_machine_setup_shows_its_label_and_its_key_without_one(self): osworld2 = self.page("osworld2") - for label in ("Docker", "Scoring programs"): + for label in ("Docker", "Scoring tools"): self.assertIn("Setup: " + label, osworld2) self.assertIn("needs " + label, osworld2) self.assertNotIn("Setup: scoring_programs", osworld2) diff --git a/app/web/builder.py b/app/web/builder.py index 001c5e6a..d3bb2d7e 100644 --- a/app/web/builder.py +++ b/app/web/builder.py @@ -94,11 +94,15 @@ def _builder_data() -> dict: gpus = constraints.of(constraints.LOCAL, strict=False).get("gpu_ids") host = {b["key"]: {n: [why] for n, why in hostreq.problems(b, list(b["host"]), gpu_ids=gpus).items()} for b in definitions.load("benchmark", every=True)[0] if b.get("host")} - # boat: why it cannot run here (its key, this machine's programs); boatKey: from a host, only the key matters here - boat_why, boat_key = boat.unusable(), boat.unusable(here=False) - data["standardHere"] = {"docker": execute.docker_problem(), "boat": boat_why, "boatKey": boat_key, - "boatSizes": {} if boat_key else execute.boat_sizes(), - "boatSpecs": {k: v for k, v in boat.SIZES.items() if v}, "licences": _licence_states(), "host": host} + # boat: why its sandboxes cannot run from here (its key, this machine's programs), else what its account + # says: the sizes it offers, each with its spec and why it cannot start one, and its VMs at once + boat_why = boat.unusable() + account = {} if boat_why else execute.boat_account() + types = account.get("types") or {} + data["standardHere"] = {"docker": execute.docker_problem(), "boat": boat_why, + "boatSizes": [{"key": k, "label": z["label"], "spec": boat.spec(k), "memory_gb": z.get("memory_gb"), + "why": types.get(k)} for k, z in boat.SIZES.items() if not types or k in types], + "boatSlots": account.get("slots"), "licences": _licence_states(), "host": host} data["csrf"] = shared._csrf_token() data["local"] = shared_local_client() data["recipesWhy"] = WRITE_HERE diff --git a/app/web/execute.py b/app/web/execute.py index 24cc9a63..f2193f52 100644 --- a/app/web/execute.py +++ b/app/web/execute.py @@ -165,19 +165,27 @@ def unpriced(models: list) -> list: return [m for m in models if m not in _PRICED["rows"]] -_BOAT_SIZES = {"at": 0.0, "key": None, "sizes": {}} +_BOAT = {"at": 0.0, "key": None, "account": {}} -def boat_sizes() -> dict: - """{size: why this account cannot start it}, from Boat's limits, kept a minute for the same key; - asked briefly: a Boat that does not answer leaves every size offered, and Run says the rest.""" +def boat_account() -> dict: + """What Boat's limits say of this account, kept a minute for the same key and asked briefly: + {types: {size: why it cannot start one, or None}, slots: VMs at once or None}. Empty without a key; + a Boat that does not answer says nothing, so every size is offered and Run says the rest.""" k = boat.key() if not k: return {} - if time.time() - _BOAT_SIZES["at"] > 60 or _BOAT_SIZES["key"] != hash(k): - types = boat.machine_types(boat.Client(k, timeout=5)) - _BOAT_SIZES.update(at=time.time(), key=hash(k), sizes={t: why for t, why in types.items() if why}) - return _BOAT_SIZES["sizes"] + if time.time() - _BOAT["at"] > 60 or _BOAT["key"] != hash(k): + client = boat.Client(k, timeout=5) + limits = client.limits() + _BOAT.update(at=time.time(), key=hash(k), account={"types": boat.machine_types(client, limits), + "slots": boat.slots(client, limits)}) + return _BOAT["account"] + + +def boat_sizes() -> dict: + """{size: why this account cannot start it}, the sizes it can start left out.""" + return {t: why for t, why in (boat_account().get("types") or {}).items() if why} def boat_size_problem(size: str): diff --git a/docs/benchmarks/supported/osworld1.md b/docs/benchmarks/supported/osworld1.md index 00426bf9..f0fda6cd 100644 --- a/docs/benchmarks/supported/osworld1.md +++ b/docs/benchmarks/supported/osworld1.md @@ -45,7 +45,7 @@ Open-ended tasks on a real Ubuntu desktop and its applications, each in a VM of | Option | Value | Refused without (`host`: the driver's machine; `embeddings`: a model the provider serves) | |---|---|---| -| `vm_provider` | `docker` | host: docker | +| `vm_provider` | `docker` | host: docker; vm: a 4 GB VM per unit | ## Good to know diff --git a/docs/benchmarks/supported/osworld2.md b/docs/benchmarks/supported/osworld2.md index 9f86f5fa..40f6c951 100644 --- a/docs/benchmarks/supported/osworld2.md +++ b/docs/benchmarks/supported/osworld2.md @@ -47,7 +47,7 @@ Long-horizon, real-world tasks on an Ubuntu desktop, across its apps, each in a | Option | Value | Refused without (`host`: the driver's machine; `embeddings`: a model the provider serves) | |---|---|---| -| `vm_provider` | `docker` | host: docker, scoring_programs | +| `vm_provider` | `docker` | host: docker, scoring_programs; vm: a 4 GB VM per unit | ## Good to know diff --git a/docs/builder.md b/docs/builder.md index 517bc0e0..0a1d1928 100644 --- a/docs/builder.md +++ b/docs/builder.md @@ -28,7 +28,7 @@ Then [read its results](results.md). | Agent (*Memory* on a memory benchmark) | Reference agents first, or **No agent · Original** for a direct-model benchmark. Memory benchmarks offer **Original memory** alongside compatible memory capabilities. Controls follow separately; each choice has its own settings tab | | Model | **Like the paper** keeps what the paper fixed: the model picker shows only documented paper models when the benchmark lists them (the filter applies to the selected agent or capability too), and the user simulator and judge are their official models. A benchmark that lists no paper models but has an official simulator or judge leaves the model to you. **Like the leaderboard**, on a benchmark that declares its leaderboard's rules, leaves the model to you and fixes what the leaderboard does: its user simulator and judge, the options it sets and at least its attempts per unit. **Any of your choice** opens the provider's full list and lets the simulator and judge be changed. Nothing fixed by the paper: the paper choice is disabled. **Served by**, a row after each model field's when the provider is OpenRouter, waits for the provider list and stays hidden when it has exactly one entry and the field has no pin; an existing pin always shows, as does an empty or failed list. Two endpoints of one provider count as two entries. It chooses the serving providers that may answer that model, tried in their order and no other ([Run a benchmark](cli/run.md#serving-providers)). Its picker is the model's: typing filters the model's providers, and each pick, or a slug typed and Enter, joins the list under it, each removable. Empty: any of them. Review shows them in its Model row | | Limits | Task limit (empty is every task), a task-name glob, concurrency, attempts per unit, retries per attempt, timeouts in minutes. On the Nullius runner, a [spending cap](cli/run.md#a-spending-cap) for the run and one per unit, turns per conversation for a benchmark that holds conversations, a [cap on model calls](cli/run.md#a-cap-on-model-calls) per unit, [time limits](cli/run.md#time-limits) per phase in minutes, forced on every participant (empty: the benchmark's or agent's own value in text colour, Nullius's default in grey), then the benchmark's own bounds in a last section of its name (τ³-bench's steps and tool errors per conversation, OSWorld's steps per task); a [list of units](cli/run.md#a-list-of-units) is set only on the command line | -| Where | This machine, a Settings host, or several machines, one shard each, merged into one job. The backend: *local processes*, as you, with the explicit yes to running unsandboxed; or *Container*, on Docker or Podman; or *Boat*, a VM per unit. A benchmark's option of where the work runs, such as OSWorld's VM provider, is asked here, one value as fixed. A benchmark option's machine setup is installed on a click (`sudo -n`), or its commands are given | +| Where | **Hosts**: this machine or Settings hosts. **Sandboxes**: Boat VMs, within the account’s sizes and capacity. Choose isolation where available; VM benchmarks keep their own VM provider | | Review | One row per decision and the prerequisites. **Run** | Original setups run on local processes or in Docker; Docker changes isolation only. diff --git a/docs/standard.md b/docs/standard.md index 6c6b3fcb..c888368b 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -181,15 +181,20 @@ A rootful Docker with `userns-remap` maps the uids of each container to sub-uids `--backend boat`: each unit's agent or capability in its own [boat.dev](https://boat.dev) VM. Driver, relay, spending cap and scoring stay with the run. -A computer-port run is refused before any unit: its server's local socket cannot be reached from the agent's VM. Choose local processes or the container backend. +Except for VM benchmarks below, computer-port runs, Original setups and checks require Hosts. + +VM benchmarks such as OSWorld run each task entirely inside Boat, including scoring. Model keys and declared benchmark credentials enter those VMs. + +For these runs, spending caps are split equally across tasks; unused shares cannot transfer. Use a dataset and limit instead of `--units` or `--shard`. | Piece | Rule | | --- | --- | | Key | Settings → Sandbox, or `BOAT_API_KEY` | -| VM | One per unit, `--boat-size` (`small` by default). Always deleted. Free trial: 2 hours at most | -| Units at a time | Capped by the account's VM limit. The run warns when it cannot start every unit | +| VM | One per unit, `--boat-size` (`small` by default). Always deleted. Free trial: 2 hours at most. VM benchmarks need more RAM than their guest VM | +| Machines | Runs from this machine; never mixed with hosts | +| Units at a time | The account's VM limit; a recipe may set fewer. The run warns when it cannot start every unit | | Agent time | Excludes the VM's start-up | -| Isolation | The agent reaches loopback only: no internet, no DNS, no model key | +| Isolation | Agent-only VMs reach loopback only: no internet, no DNS, no model key. VM benchmarks isolate the whole task | | Files | A link into your home or keys refuses the unit. Links out of the workspace are not copied back | | Code checks | A `runs_code` check needs Docker where the run runs | | Not run | Units needing the public web, their own image, a licence file or server of this machine, or MCP servers. No GPUs | @@ -489,7 +494,7 @@ The upstream's dependencies go in `driver.install` (a capability's `install` is | `datasets` | The datasets it applies to | | `about` | A line about each value | | `names` | A name for some values, shown in their place; a recipe and the CLI keep the value | -| `needs` | What a value needs: an `embeddings` model, a `reranker` model, `host` requirements, or named `files` | +| `needs` | What a value needs: an `embeddings` model, a `reranker` model, `host` requirements, named `files`, or on a Where option the `vm` each unit boots (`{"memory_gb": 4}`) | | `when` | Another option's values that make it apply | | `limit` | `true`: a bound of the upstream's own (steps, errors). The builder shows it in Limits, under the benchmark's name | | `step` | `where`: where the work runs, such as the VMs' provider, asked on Where, one value as fixed; `dataset`: what its datasets need, asked on Dataset for the datasets it applies to |