diff --git a/app/nullius/catalogue/agents/codex/agent.json b/app/nullius/catalogue/agents/codex/agent.json index eca12acd..73ac23c3 100644 --- a/app/nullius/catalogue/agents/codex/agent.json +++ b/app/nullius/catalogue/agents/codex/agent.json @@ -141,13 +141,13 @@ }, "agent.computer/v1": { "passed": "4/4", - "on": "2026-11-17", + "on": "2026-11-19", "model": "gpt-6-luna", "provider": "openai", "version": "2.2.2", - "kit_version": 1, + "kit_version": 2, "backend": "container", - "job": "2026-11-17__12-53-08__kit__agent-computer-v1__codex-2.2.2", + "job": "2026-11-19__09-44-46__kit__agent-computer-v1__codex-2.2.2", "repeat": 3, "checks": { "see": "3/3", @@ -171,6 +171,10 @@ "batch": { "passed": "3/3", "ok": true + }, + "late": { + "passed": "3/3", + "ok": true } } } diff --git a/app/nullius/catalogue/benchmarks/osworld1/benchmark.json b/app/nullius/catalogue/benchmarks/osworld1/benchmark.json index 9377167d..c79eacef 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/benchmark.json +++ b/app/nullius/catalogue/benchmarks/osworld1/benchmark.json @@ -9,7 +9,7 @@ ], "origin": "official", "good_to_know": [ - "An agent runs it through the computer port: the agent's tool calls drive OSWorld's own loop, one step for each acting call. Steps (100) and Pause (3.0 seconds) set that loop.", + "An agent runs it through the computer port: the agent's tool calls drive OSWorld's own loop, one step for each acting call. A step's observation is the screenshot that loop hands its agent for the next decision, with no capture added; the files a step names are read once its actions and their pauses have run. Steps (100) and Pause (3.0 seconds) set that loop.", "Original runs one of OSWorld's own agents, chosen by the Agent option (M3 by default), with a model it is published with.", "Providers: OpenAI for OpenAI CUA; OpenAI-compatible for Muse Spark (Meta's API) and for EvoCUA, AutoGLM-OS and Mano (your model server); OpenRouter for the others.", "Only M3, PromptAgent and Qwen-CUA have completed tasks through Nullius. The other agents were checked up to their first model call.", diff --git a/app/nullius/catalogue/benchmarks/osworld1/driver.py b/app/nullius/catalogue/benchmarks/osworld1/driver.py index 53f7ca22..e38b979e 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/driver.py +++ b/app/nullius/catalogue/benchmarks/osworld1/driver.py @@ -20,6 +20,9 @@ the MCP server computer (the port's pipe, joined to the runner's socket) and the agent's time. The options max_steps and pause are the loop's +task_given - an agent's unit (driver.task_given): the agent is about to be + given its task. The runner is told on its lifeline, closes the + port's window before the task and says so; this answers then answer - a setup's (original, model): the script of the run's agent option (agents.json: the upstream's scripts/python/run_*.py) as published, on the one task (a test_all_meta file naming it), with the arguments @@ -40,8 +43,8 @@ waits for the runner to end and reads result.txt from the runner's folder, never the workspace, and only from a runner this driver started that ended by itself with status 0, through folders opened - without following a link. It copies result.txt and the call log to - the trial's verifier/osworld/. + without following a link. It copies result.txt, the call log and the + record of prepare calls to the trial's verifier/osworld/. For the null control, nop.py: the upstream's run_single_example with an agent that answers DONE at its first step, its result.txt read the same way @@ -120,6 +123,7 @@ REMOVE_SECONDS = 30 # the docker commands that remove a unit's VMs, together FALLBACK_SECONDS = 2 # this driver's removal when no reaper could start: inside the core's 3 s RUNNERS = {} # the runner folder of each unit this driver set up -> its process +GIVEN_SECONDS = 30 # the runner's word that the window before the task is closed def _load(path): @@ -473,7 +477,9 @@ def _start_runner(unit, dir, instruction, until): argv = [sys.executable, RUNNER_FILE, "--example", task, "--result-dir", os.path.join(where, RESULTS), "--socket", os.path.join(sock_dir, "s"), "--ready", ready, "--lifeline", str(life_r), "--steps", "100" if steps == "default" else steps, "--pause", str(_option("pause", "3.0", float)), - "--vm", vm, "--calls", os.path.join(where, "computer-calls.jsonl"), "--held", held] + "--vm", vm, "--calls", os.path.join(where, "computer-calls.jsonl"), "--held", held, + "--prepared", os.path.join(where, "computer-prepare.jsonl"), + "--given", os.path.join(sock_dir, "given.json")] lock = open(START_LOCK, "a") while True: # VM starts one after another, as the setups' do try: @@ -575,6 +581,26 @@ def _turn_end(entry): pass +def task_given(unit, dir, **_): + """The agent is about to be given its task: the runner closes the port's window before it and says so in the + socket's folder; this answers once it has, within GIVEN_SECONDS.""" + entry = RUNNERS.get(_runner_dir(dir)) + if entry is None or entry.get("life") is None: + raise RuntimeError("no runner of this driver's for this unit: the window before the task cannot be closed") + said = os.path.join(entry["sock_dir"], "given.json") + os.write(entry["life"], _port().TASK_GIVEN + b"\n") + t0 = time.time() + while not os.path.exists(said): + if _exited(entry["p"]) is not None or time.time() - t0 > GIVEN_SECONDS or CLOSED.wait(0.05): + raise RuntimeError("the runner did not say that the window before the task is closed") + with open(said, encoding="utf-8") as f: + ok = json.load(f).get("ok") + os.unlink(said) + if not ok: + raise RuntimeError("the runner could not close the window before the task: " + str(ok)) + return {} + + def _reap_runner(where, seconds=None): """The unit's runner let go, given ``seconds`` (REAP_SECONDS) to end by itself, its whole group killed while its leader is still unreaped, then reaped; the VM containers of the unit's image link removed within REMOVE_SECONDS, @@ -772,13 +798,16 @@ def _agent_score(where, dir, domain, eid): value = float(result.decode().strip()) except (OSError, ValueError, RuntimeError) as e: raise RuntimeError(f"the runner's result could not be read ({e}): " + _tail(os.path.join(where, "runner.log"))) - try: - calls = _plain(trial, ".work", "osworld", "computer-calls.jsonl") - except (OSError, RuntimeError): - calls = b"" + kept = {"computer-calls.jsonl": b""} + for name in ("computer-calls.jsonl", "computer-prepare.jsonl"): # the record of prepare calls when there is one + try: + kept[name] = _plain(trial, ".work", "osworld", name) + except (OSError, RuntimeError): + pass + calls = kept["computer-calls.jsonl"] out = os.path.join(trial, "verifier", "osworld") os.makedirs(out, exist_ok=True) - for name, data in (("result.txt", result), ("computer-calls.jsonl", calls)): + for name, data in [("result.txt", result)] + sorted(kept.items()): with open(os.path.join(out, name), "wb") as f: f.write(data) return {"reward": value, "detail": {"result": value, "by": "runner.py, run_single_example, DesktopEnv.evaluate", @@ -816,7 +845,7 @@ def score(unit, dir, transcript=None, answer=None, setup=None, **_): return _scored(where, "nop.py, DesktopEnv.evaluate") -HANDLERS = {"units": units, "setup": setup, "answer": answer, "score": score, "end": end} +HANDLERS = {"units": units, "setup": setup, "task_given": task_given, "answer": answer, "score": score, "end": end} def main(): diff --git a/app/nullius/catalogue/benchmarks/osworld1/runner.py b/app/nullius/catalogue/benchmarks/osworld1/runner.py index 3c8c68e7..c759e7b8 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/runner.py +++ b/app/nullius/catalogue/benchmarks/osworld1/runner.py @@ -1,21 +1,24 @@ """One unit of an agent on OSWorld1: the upstream's DesktopEnv and its own run_single_example, imported and called as -they are, with the computer port's bridge as the loop's agent (nullius.standard.computer_port). +they are, with the computer port's bridge as the loop's agent (nullius.standard.computer_port, agent.computer/v1). runner.py --example TASK_JSON --result-dir DIR --socket PATH --ready FILE --lifeline FD --steps N --pause SECONDS - --vm VM_IMAGE --calls FILE [--held FOLDER] + --vm VM_IMAGE --calls FILE [--held FOLDER] [--prepared FILE] [--given FILE] The driver starts it in an agent's unit's setup, in the unit's runner folder (the trial's .work/osworld/), with the upstream's folder and the nullius package on its import path. It builds the upstream's DesktopEnv as the null control does (nop.py), serves the MCP server computer at PATH (a Unix socket in a folder of the user's alone, which the driver made) over a Desktop of five methods on the upstream's controllers, and calls run_single_example with the bridge as the loop's agent, N as max_steps and SECONDS as sleep_after_execution. No file of the upstream is edited, patched or -replaced. +replaced. The loop is handed the DesktopEnv through a view (Watched) that keeps the screenshot of the observation each +env.step returns: a step's observation is the one the loop hands its agent for the next decision, with no capture +added. The agent's folder in the VM is FOLDER. - READY, a file of mode 0600: {"ok": true, "password": the desktop user's} once the loop asks for the agent's first step (after the task's setup and the upstream's own wait); {"ok": false, "error": ...} when the unit fails before. - FD, the read end of a pipe from the driver: a line turn-end ends the agent's turn (the loop then gets no action for - the steps left and evaluates the desktop as it is); its close stops the runner as SIGTERM does. -- FILE, the call log: one line per tool call, {tool, start, end, isError}. + the steps left and evaluates the desktop as it is); a line task-given closes the port's window before the task, and + the runner then writes --given's file, {"ok": true}; its close stops the runner as SIGTERM does. +- FILE, the call log: one line per tool call, {tool, start, end, isError}; --prepared's, one line per prepare call. - FOLDER, the run's folder of files sent once, when the driver gives one (computer_port.Held). A stop is taken once. The clean-up (the bridge's end and the server's close, then the desktop's close within @@ -33,17 +36,21 @@ import threading SCREEN = (1920, 1080) -PYTHON_SECONDS = 120 # the upstream controller's wait on a command's reply FINISH_SECONDS = 5 # the bridge's end and the server's close CLOSE_SECONDS = 25 # the desktop's close: the provider stops and removes its container STOP_SECONDS = FINISH_SECONDS + CLOSE_SECONDS +FOLDER = "/tmp/agent" # the agent's own folder in the VM class Desktop: - """The port's five operations over the upstream's own controllers.""" + """The port's five operations over the upstream's own controllers, and the observation of the step the loop just + ran (``shot``, kept by Watched).""" def __init__(self, env, scratch): - self.env, self.scratch = env, scratch + self.env, self.scratch, self.shot = env, scratch, None + + def observation(self): + return self.shot def python(self, code): return self.env.controller.execute_python_command(code) @@ -58,7 +65,7 @@ def upload(self, path, data): os.unlink(local) def read(self, path): - return self.env.controller.get_file(path) + return self.env.controller.get_file(path, strict=True) # a missing file at once, with no retry def capture(self): png = self.env.controller.get_screenshot() @@ -70,14 +77,42 @@ def password(self): return self.env.client_password +class Watched: + """The upstream's DesktopEnv as its loop is handed it: the env itself for every attribute, each env.step's + observation's screenshot also kept in the Desktop, for the bridge.""" + + def __init__(self, env, desktop): + object.__setattr__(self, "_env", env) + object.__setattr__(self, "_desktop", desktop) + + def step(self, *args, **kwargs): + out = self._env.step(*args, **kwargs) + obs = out[0] if isinstance(out, tuple) and out else None + self._desktop.shot = obs.get("screenshot") if isinstance(obs, dict) else None + return out + + def __getattr__(self, name): + return getattr(self._env, name) + + def __setattr__(self, name, value): + setattr(self._env, name, value) + + class Turn: - """What the lifeline's turn-end reaches: the bridge, once it exists.""" - bridge = None + """What the lifeline reaches: the bridge, once it exists, and the file that says the window before the task is + closed (``given``, written by ``said``).""" + bridge = given = said = None def end_turn(self): if self.bridge is not None: self.bridge.end_turn() + def close_window(self): + if self.bridge is not None: + self.bridge.close_window() + if self.given and self.said: + self.said(self.given, ok=self.bridge is not None) + def close_within(env, seconds): """The desktop closed in a thread of its own, waited for ``seconds``: True when it closed in that time.""" @@ -97,13 +132,15 @@ def main(argv=None): p = argparse.ArgumentParser() for name in ("--example", "--result-dir", "--socket", "--ready", "--vm", "--calls"): p.add_argument(name, required=True) - p.add_argument("--held") + for name in ("--held", "--prepared", "--given"): + p.add_argument(name) p.add_argument("--lifeline", type=int, required=True) p.add_argument("--steps", type=int, required=True) p.add_argument("--pause", type=float, required=True) a = p.parse_args(argv) from nullius.standard import computer_port as port stop, turn = port.Stop(STOP_SECONDS), Turn() + turn.given, turn.said = a.given, port.ready try: # the clean-up holds before the lifeline can stop the runner stop.install() threading.Thread(target=port.lifeline, args=(a.lifeline, turn), name="lifeline", daemon=True).start() @@ -139,11 +176,12 @@ def first(): # the loop asks for the agent's headless=True, os_type="Ubuntu", require_a11y_tree=False, client_password="password", enable_proxy=False) os.makedirs(where, exist_ok=True) - bridge = port.Bridge(Desktop(env, os.path.dirname(os.path.abspath(a.calls))), a.steps, a.pause, SCREEN, - PYTHON_SECONDS, on_first=first, held=port.Held(a.held) if a.held else None) + desktop = Desktop(env, os.path.dirname(os.path.abspath(a.calls))) + bridge = port.Bridge(desktop, a.steps, a.pause, SCREEN, FOLDER, on_first=first, + held=port.Held(a.held) if a.held else None, window=bool(a.given), prepared=a.prepared) server = port.Server(bridge, a.socket, a.calls) turn.bridge = bridge - lib_run_single.run_single_example(bridge, env, example, a.steps, example["instruction"], + lib_run_single.run_single_example(bridge, Watched(env, desktop), example, a.steps, example["instruction"], argparse.Namespace(sleep_after_execution=a.pause, result_dir=a.result_dir), where, []) except SystemExit: # SIGTERM or the lifeline's end: the unit was stopped diff --git a/app/nullius/catalogue/benchmarks/osworld1/standard.json b/app/nullius/catalogue/benchmarks/osworld1/standard.json index 0b588a1e..5eb63b92 100644 --- a/app/nullius/catalogue/benchmarks/osworld1/standard.json +++ b/app/nullius/catalogue/benchmarks/osworld1/standard.json @@ -27,7 +27,8 @@ "locked": true }, "participant": true, - "unit_end": true + "unit_end": true, + "task_given": true }, "mcp_servers": { "computer": { diff --git a/app/nullius/standard/computer_port.py b/app/nullius/standard/computer_port.py index 8135cbe7..33de6a1c 100644 --- a/app/nullius/standard/computer_port.py +++ b/app/nullius/standard/computer_port.py @@ -5,6 +5,13 @@ which has the loop's ``reset`` and ``predict``. Standard library alone, so that a runner in the upstream's own Python imports it with the package's folder on its path. +Once the agent is given its task, the desktop is reached through acting calls alone: ``screenshot`` returns the +observation the last step fixed (``Bridge.seen``), taken from the benchmark's loop or, for a loop that hands none, as +a picture when the step's last pause has passed; a ``step`` names the files of the agent's folder it wants back, read +once its actions and pauses have run. Before the task, in a window that the benchmark's side closes when the run gives +it (``Bridge.close_window``), ``upload`` writes inside that folder and ``prepare`` runs actions there and reads files +back, neither a step; an acting call is refused until the task is given. + - ``Desktop``: the five operations an entry or a kit gives over its desktop. - The tools: one table (``tools``) gives both what ``tools/list`` says and what ``checked`` accepts. A call is checked in full, its upload's data decoded, before it is queued; a call refused there takes no step and never reaches the @@ -58,6 +65,8 @@ TURN_END = b"turn-end" # the lifeline's one message: the agent's turn has ended APPEND = ("import os\ntry:\n s = open({part!r}, 'rb'); d = open({path!r}, 'ab'); d.write(s.read()); d.close(); " "s.close()\nfinally:\n os.remove({part!r})\nprint(os.path.getsize({path!r}))") +MAKE_FOLDER = ("import os, shutil\np = {folder!r}\nif os.path.islink(p) or (os.path.lexists(p) and not os.path.isdir(p)):\n" + " os.remove(p)\nshutil.rmtree(p, ignore_errors=True)\nos.makedirs(p, 0o700)\nprint('made')") class Desktop(Protocol): @@ -65,8 +74,8 @@ class Desktop(Protocol): a time, from the thread of the benchmark's loop.""" def python(self, code: str) -> Optional[dict]: - """Python code run in the desktop as its user: the reply ``{status, output, error, returncode}``, or None when - the desktop gave none in time.""" + """Python code run in the desktop as its user (a prepare's actions, the agent's folder made, an upload's + chunk added): the reply ``{status, output, error, returncode}``, or None when the desktop gave none in time.""" def upload(self, path: str, data: bytes) -> None: """The file at ``path`` in the desktop made to hold ``data``.""" @@ -80,10 +89,18 @@ def capture(self) -> bytes: def password(self) -> str: """The desktop user's password.""" + # The bridge also asks, when the desktop has it: observation() -> bytes, the screenshot the benchmark's own loop + # took when the step it just ran ended (for a loop that keeps one). Without it, the bridge takes a picture by + # capture(). + # -- the tools: one table for what tools/list says and what a call may hold ------------------------------------------ -NAMES = ["info", "screenshot", "read", "python", "upload", "click", "move", "drag", "scroll", "type", "key", "wait", - "step", "done", "fail"] +NAMES = ["info", "screenshot", "upload", "prepare", "click", "move", "drag", "scroll", "type", "key", "wait", "step", + "done", "fail"] +WINDOW = ("upload", "prepare") # the tools of the window before the task +FILES_ROOM = LINE_BYTES - 64 * 1024 # of an answer, what its files and image may take +TASK_GIVEN = b"task-given" # the lifeline's message: the agent is being given its task +PREPARE = "import pyautogui\nimport time\npyautogui.FAILSAFE = False\n{code}" # a prepare's action, as a step's is SCREEN_TOOLS = ("click", "move", "drag", "scroll", "type", "key", "wait") # answered with the screen after the pause ACTING = SCREEN_TOOLS + ("step", "done", "fail") FINAL = {"done": "DONE", "fail": "FAIL"} @@ -102,6 +119,10 @@ def password(self) -> str: PATTERN_WORDS = {NOT_BLANK: "is not blank", SHA256: "is 64 lowercase hexadecimal digits", BASE64: "is base64"} +FILES = {"type": "array", + "items": {"type": "string", "minLength": 1, "description": "A path inside folder"}} + + @functools.lru_cache(maxsize=8) def tools(screen): """The tools for a screen of ``screen`` (width, height) pixels: [(name, description, input schema)].""" @@ -118,23 +139,32 @@ def tools(screen): data = {"type": "string", "contentEncoding": "base64", "pattern": BASE64, "maxLength": B64_CHARS} if B64_LAST: data["anyOf"] = [{"maxLength": B64_CHARS - 4}, {"pattern": B64_LAST}] + actions = {"type": "array", "items": { + "type": "string", "pattern": NOT_BLANK, + "not": {"pattern": f"^{BLANK}*(DONE|FAIL){BLANK}*$", + "description": "the task's end is the tool done or fail, or the argument end, not an action of step"}}} return [ - ("info", "The screen's size, the steps in all and left, the pause after each action in seconds, the seconds " - "python waits for the desktop's reply, and the desktop user's password.", obj({})), - ("screenshot", "The screen now, as a PNG image at its own size. Look at it before the first pointing action.", obj({})), - ("read", "A file of the desktop: data is its content in base64, or null when there is no such file.", - obj({"path": text}, ["path"])), - ("python", "Runs Python code in the desktop as its user and returns its reply: status, output, error, " - "returncode. For reading the desktop and for your own programs there; pointer and keyboard input " - "goes through the acting tools.", obj({"code": text}, ["code"])), - ("upload", "Writes data to a file of the desktop and returns the file's size so far. data is base64, at most " - "4 MiB a call; append false starts the file, true adds to it. sha256, optional, is the whole " + ("info", "The screen's size, the steps in all and left, the pause after each action in seconds, the desktop " + "user's password, and folder: a folder of the desktop that is yours, empty at the start, for upload " + "and for the files a step returns.", obj({})), + ("screenshot", "The observation of the last step, as a PNG image: the screen as the benchmark took it when " + "that step ended, or at the start before the first step. It takes no new picture; wait is the " + "way to look again. Look at it before the first pointing action.", obj({})), + ("upload", "Before the task alone, in the window between the session's start and the task: writes data to a " + "file inside folder (see info) and returns the file's size so far. Not a step. data is base64, at " + "most 4 MiB a call; append false starts the file, true adds to it. sha256, optional, is the whole " "file's: when the run already holds a file of that sha256, it may be given with no data, and it " "starts the file.", obj({"path": text, "data": data, "append": {"type": "boolean"}, "sha256": {"type": "string", "pattern": SHA256}}, ["path"], anyOf=[{"required": ["data"]}, {"required": ["sha256"]}], **{"if": {"not": {"required": ["data"]}}, "then": {"properties": {"append": {"const": False}}}})), + ("prepare", "Before the task alone, in the window between the session's start and the task: runs actions in " + "the desktop, each a line of Python with pyautogui and time imported as a step's are, with no " + "pause after them, then reads files, any number of paths inside folder. Returns {files}: each in " + "base64, null when there is no such file. Not a step.", + {"type": "object", "additionalProperties": False, "required": ["actions"], + "properties": {"actions": actions, "files": FILES}}), ("click", "One step: a click at the point. Returns the screen as a PNG image after the pause.", obj({"x": x, "y": y, "button": button, "count": {"type": "integer", "minimum": 1, "maximum": 3, "description": "Default 1"}}, ["x", "y"])), @@ -157,16 +187,14 @@ def tools(screen): ("wait", "One step: no action, the pause alone. Returns the screen as a PNG image after the pause.", obj({})), ("step", "One step of several actions, run in order: each is a line of Python run in the desktop with " "pyautogui and time imported, or WAIT. An empty list is a step with no action. With end, done or " - "fail, the actions run and then the task ends, in this one step. Returns the screen as a PNG image " - "after the actions and pauses when screenshot is true.", - obj({"actions": {"type": "array", "items": { - "type": "string", "pattern": NOT_BLANK, - "not": {"pattern": f"^{BLANK}*(DONE|FAIL){BLANK}*$", - "description": "the task's end is the tool done or fail, or the argument end, not an action " - "of step"}}}, - "response": {"type": "string"}, "screenshot": {"type": "boolean"}, + "fail, the actions run and then the task ends, in this one step. files, any number of paths inside " + "folder, are read once the actions and their pauses have run: the answer's files holds each in " + "base64, null when there is no such file. Returns the step's observation as a PNG image when " + "screenshot is true.", + obj({"actions": actions, "response": {"type": "string"}, "screenshot": {"type": "boolean"}, "end": {"type": "string", "enum": ["done", "fail"], - "description": "Once the actions have run: done, the task is complete; fail, it cannot be done"}}, + "description": "Once the actions have run: done, the task is complete; fail, it cannot be done"}, + "files": FILES}, ["actions"])), ("done", "The last step: the task is complete. Returns {steps_left, done}, with no image.", obj({"message": {"type": "string"}})), @@ -179,6 +207,22 @@ def listed(screen) -> list: return [{"name": n, "description": d, "inputSchema": s} for n, d, s in tools(tuple(screen))] +def _framed(named, got): + """The bytes one file takes in an answer: its entry in the files object, framed once more as the JSON text of the + answer's content (each quote and backslash escaped).""" + text = json.dumps({named: got}) + return len(text) + text.count('"') + text.count("\\") + + +def inside(folder, path, name): + """``path``, a path of the desktop inside ``folder`` (absolute, or relative to the folder), as an absolute path; + refused when it is not one, or not written in its plain form.""" + full = path if path.startswith("/") else posixpath.join(folder, path) + if "\x00" in full or posixpath.normpath(full) != full or not full.startswith(folder.rstrip("/") + "/"): + raise Refused(f"{name} is a path inside {folder}, with no . or .. part") + return full + + class Refused(Exception): """A call answered with ``isError`` and these words.""" @@ -258,6 +302,7 @@ def __init__(self, tool, args, reply): self.start = time.time() self.clean, self.actions, self.response, self.screen, self.data = {}, [], "", False, None self.end = None + self.files = None # a step's or a prepare's files, [(as named, the desktop's path)] self.answered = False self._lock = threading.Lock() @@ -269,12 +314,19 @@ def answer(self, value=None, image=None, error=None): self._reply(self, value, image, error) -def checked(call, screen) -> None: +def checked(call, screen, folder) -> None: """The call checked against its tool's schema, in full and before it is queued; for an acting call, the actions it stands for (Python lines run with pyautogui and time imported, or WAIT, DONE, FAIL). Refused when it does not - hold: the call is then answered at once, takes no step and never reaches the desktop.""" + hold: the call is then answered at once, takes no step and never reaches the desktop. An upload's path and the + files of a step or a prepare are also checked to be inside ``folder``, the agent's.""" schema = next(s for n, _, s in tools(tuple(screen)) if n == call.tool) a = call.clean = _valid(call.args, schema, call.tool, "arguments") + if call.tool == "upload": + a["path"] = inside(folder, a["path"], "path") + if call.tool in ("step", "prepare") and "files" in a: + call.files = [(p, inside(folder, p, f"files[{i}]")) for i, p in enumerate(a["files"])] + if call.tool == "prepare": + call.actions = list(a["actions"]) tool, button = call.tool, repr(a.get("button", "left")) if tool == "click": call.actions = [f"pyautogui.click({a['x']}, {a['y']}, clicks={a.get('count', 1)}, button={button})"] @@ -351,13 +403,24 @@ class Bridge: """The benchmark loop's agent, over a Desktop. ``steps`` is the loop's budget, ``pause`` the seconds it waits after each action, ``screen`` the screen's size in - pixels, ``python_seconds`` how long the desktop waits for a reply to python. ``on_first`` is called at the loop's - first ``predict``, before any call is served (a runner writes its ready file there). ``held`` is a Held, or None.""" - - def __init__(self, desktop, steps, pause, screen, python_seconds, on_first=None, held=None): + pixels, ``folder`` the agent's own folder in the desktop (an absolute path), made empty at the loop's first + ``predict``. ``on_first`` is called at the loop's first ``predict``, before any call is served (a runner writes + its ready file there). ``held`` is a Held, or None. The window before the task is open when ``window`` is true + (the benchmark's side is told when the task is given, and calls ``close_window``); with no window, upload and + prepare are refused from the start. ``prepared``, a file, gets one JSON line per prepare call: its actions and the + files it names, before they run.""" + + def __init__(self, desktop, steps, pause, screen, folder, on_first=None, held=None, window=False, prepared=None): self.desktop, self.steps, self.pause = desktop, int(steps), pause - self.screen, self.python_seconds = (int(screen[0]), int(screen[1])), python_seconds + self.screen = (int(screen[0]), int(screen[1])) self.on_first, self.held = on_first, held + if not (isinstance(folder, str) and folder.startswith("/") and len(folder) > 1 + and posixpath.normpath(folder) == folder): + raise ValueError("the agent's folder is an absolute path in its plain form") + self.folder = folder + self.window, self.prepared = bool(window), prepared # the window before the task + self.preparing = False + self.seen, self.unseen = None, "no step has ended" # the observation, or why there is none self.password = desktop.password() self.used = 0 # acting calls handed to the loop: its steps self.booked = 0 # acting calls taken in, handed or waiting @@ -378,8 +441,11 @@ def predict(self, instruction, obs): Once the turn has ended, no action, but for a done or fail taken before it.""" if self.first: self.first = False + self._begin(obs) if self.on_first: self.on_first() + else: + self._handed(obs) while True: call = self._next() if call is None: @@ -392,6 +458,94 @@ def predict(self, instruction, obs): return call.response, list(call.actions) return call.response, self._run(call) + # -- the agent's folder and the observation ------------------------------------------------------------------- + def _begin(self, obs): + """The loop's first predict: the agent's folder made empty, and the first observation, the one the loop + hands or, when it hands none, a picture taken now.""" + reply = self.desktop.python(MAKE_FOLDER.format(folder=self.folder)) + if not isinstance(reply, dict) or str(reply.get("output", "")).strip() != "made": + raise RuntimeError(f"the agent's folder {self.folder} could not be made: {json.dumps(reply)[:300]}") + if not self._handed(obs): + self.seen = self.desktop.capture() + + def _handed(self, obs): + """The screenshot of the observation the loop hands its agent, kept when there is one.""" + shot = obs.get("screenshot") if isinstance(obs, dict) else None + if isinstance(shot, (bytes, bytearray)) and shot: + self.seen, self.unseen = bytes(shot), "" + return True + return False + + def _observe(self): + """The observation of a step that ran an action, fixed as it ends: the benchmark's own when its desktop keeps + one (``observation``), else a picture taken now, the step's last pause passed.""" + look = getattr(self.desktop, "observation", None) + shot = look() if look is not None else self.desktop.capture() + if not shot: + raise RuntimeError("the benchmark gave no observation") + return bytes(shot) + + def _step_end(self, call): + """In the loop's thread, once the step's actions and their pauses have run: its observation fixed, when it + ran an action, and the files it names read. The files' answer, or None when it names none.""" + if call.actions: + try: + self.seen, self.unseen = self._observe(), "" + except Exception as e: # the step was run all the same + self.seen, self.unseen = None, f"{type(e).__name__}: {e}" + if call.files is None: + return None + return self._files(call.files, self.seen if call.screen else None) + + def _files(self, named_paths, image=None): + """Each file read, in order, as long as the answer holds it: one that would take the answer past LINE_BYTES, + with ``image`` in it, is an error, and the files after it are still tried.""" + room, out = FILES_ROOM - (4 * -(-len(image) // 3) if image else 0), {} + for named, path in named_paths: + got = self._file(path) + if isinstance(got, str) and _framed(named, got) > room: + got = {"error": f"the answer would pass {LINE_BYTES} bytes with this file"} + room -= _framed(named, got) + out[named] = got + return out + + def _file(self, path): + try: + data = self.desktop.read(path) + except Refused as e: + return {"error": str(e)} + except Exception as e: # that file alone: the step has run + return {"error": f"the desktop could not read it: {type(e).__name__}: {e}"} + if data is None: + return None + if len(data) > READ_BYTES: + return {"error": f"over {READ_BYTES} bytes"} + return base64.b64encode(data).decode() + + def _prepare(self, call): + """A prepare call, in the loop's thread: its record written, its actions run one at a time with no pause (WAIT + does nothing), then its files read. An action the desktop did not end in time stops it.""" + if self.prepared: + with open(self.prepared, "a", encoding="utf-8") as f: + f.write(json.dumps({"start": time.time(), "actions": call.actions, + "files": [named for named, _ in call.files or []]}) + "\n") + for i, code in enumerate(call.actions): + if code.strip() != "WAIT" and self.desktop.python(PREPARE.format(code=code)) is None: + raise Refused(f"the desktop did not end actions[{i}] in time; the actions after it were not run") + return {"files": self._files(call.files or [])} + + def close_window(self): + """Close the window before the task, refuse waiting preparation and wait for active preparation to end.""" + with self.change: + self.window = False + refused = [c for c in self.calls if c.tool in WINDOW] + self.calls = collections.deque(c for c in self.calls if c not in refused) + for call in refused: + call.answer(error=f"the task has been given: {call.tool} is taken only before it") + with self.change: + while self.preparing: + self.change.wait() + def _next(self): refused = [] with self.change: @@ -428,6 +582,7 @@ def _run(self, call): yield FINAL[call.end] def _ran(self, call, ran): + files = self._step_end(call) if ran and not self.turn_ended else None with self.change: self.running = None if ran and call.end: # the task has ended, as after done or fail @@ -439,35 +594,41 @@ def _ran(self, call, ran): if not ran: call.answer(error="the agent's turn has ended" if turn_ended else "the step was not run to its end") else: - try: - call.answer(value, self.desktop.capture() if call.screen and not turn_ended else None) - except Exception as e: # the screen could not be taken: the step itself was run - call.answer(error=f"the step was run; the screen could not be taken: {type(e).__name__}: {e}") + if files is not None: + value["files"] = files + if call.screen and not turn_ended and self.seen is None: + call.answer(error=f"the step was run; the screen could not be taken: {self.unseen}") + else: + call.answer(value, self.seen if call.screen and not turn_ended else None) for c in refused: c.answer(error=self.ended) def _serve(self, call): + preparing = False try: + if call.tool in WINDOW: + with self.change: + if not self.window: + raise Refused(f"the task has been given: {call.tool} is taken only before it") + self.preparing = preparing = True call.answer(**self._reading(call)) except Refused as e: call.answer(error=str(e)) except Exception as e: # the desktop failed it: its words, and the agent may call again call.answer(error=f"the desktop failed {call.tool}: {type(e).__name__}: {e}") + finally: + if preparing: + with self.change: + self.preparing = False + self.change.notify_all() def _reading(self, call): - a = call.clean if call.tool == "screenshot": - return {"image": self.desktop.capture()} - if call.tool == "read": - data = self.desktop.read(a["path"]) - if data is not None and len(data) > READ_BYTES: - raise Refused(f"{a['path']} is over {READ_BYTES} bytes") - return {"value": {"data": None if data is None else base64.b64encode(data).decode()}} - if call.tool == "python": - reply = self.desktop.python(a["code"]) - if reply is None: - raise Refused(f"the desktop gave no reply within {self.python_seconds:g} s") - return {"value": reply} + if self.seen is None: + raise Refused(f"there is no observation: {self.unseen}") + return {"image": self.seen} + if call.tool == "prepare": + return {"value": self._prepare(call)} return {"value": self._upload(call)} def _upload(self, call): @@ -530,7 +691,12 @@ def submit(self, call): call once every step is taken.""" with self.change: refusal = self.ended - if refusal is None and call.tool in ACTING: + if refusal is None and self.window and call.tool in ACTING: + refusal = ("the task has not been given yet: before it, the desktop takes info, screenshot, upload " + "and prepare, and no step") + elif refusal is None and not self.window and call.tool in WINDOW: + refusal = f"the task has been given: {call.tool} is taken only before it" + elif refusal is None and call.tool in ACTING: if self.booked >= self.steps: refusal = "the steps are used up" else: @@ -545,20 +711,25 @@ def info(self): with self.change: left = self.steps - self.used return {"screen": list(self.screen), "steps": self.steps, "steps_left": left, "pause": self.pause, - "python_seconds": self.python_seconds, "password": self.password} + "password": self.password, "folder": self.folder} def instructions(self): w, h = self.screen - return (f"A desktop seen in screenshots and driven by pointer and keyboard actions. The screen is {w} by {h} " - f"pixels. Points use x and y in screenshot pixels at the screen's size, from its top left corner. " - f"Look at a screenshot before the first pointing action. Each call of click, move, drag, scroll, " - f"type, key, wait, step, done or fail is one step. After the actions and their {self.pause:g} s pauses, " - f"click, move, drag, scroll, type, key and wait return the screen as a PNG image; step does when " - f"screenshot is true. done and fail answer before they run, with no image. A step with end, done " - f"or fail, runs its actions and then ends the task, in that one step. " - f"There are {self.steps} steps in all, and steps_left in each acting answer. Once they are used " - f"up, every call but info is refused. Pointer and keyboard input goes through those tools, not through " - f"python. A turn that ends with neither done nor fail ends the task as the desktop then is.") + return (f"A desktop seen in screenshots and driven by pointer and keyboard actions. The screen is {w} by " + f"{h} pixels. Points use x and y in screenshot pixels at the screen's size, from its top left " + f"corner. Look at a screenshot before the first pointing action. Each call of click, move, drag, " + f"scroll, type, key, wait, step, done or fail is one step, and the desktop is reached through " + f"these calls alone. After the actions and their {self.pause:g} s pauses, click, move, drag, " + f"scroll, type, key and wait return the step's observation, the screen as a PNG image; step does " + f"when screenshot is true, and returns the files of folder it names. screenshot returns the last " + f"step's observation and takes no new picture: wait is the way to look again. Before the task is " + f"given, in the window from the session's start, upload and prepare work inside folder " + f"({self.folder}) and take no step, and no acting call is taken; once it is given, upload and " + f"prepare are refused. done and fail answer before " + f"they run, with no image. A step with end, done or fail, runs its actions and then ends the " + f"task, in that one step. There are {self.steps} steps in all, and steps_left in each acting " + f"answer. Once they are used up, every call but info is refused. A turn that ends with neither " + f"done nor fail ends the task as the desktop then is.") def end_turn(self): """The agent's turn has ended: the wait for its next call ends, the actions of a step not yet run are not @@ -807,7 +978,8 @@ def handle(self, m, wire): return None if method == "initialize": result = {"protocolVersion": params.get("protocolVersion") or PROTOCOL, "capabilities": {"tools": {}}, - "serverInfo": {"name": "computer", "version": "1"}, "instructions": self.bridge.instructions()} + "serverInfo": {"name": "computer", "version": "1"}, + "instructions": self.bridge.instructions()} elif method == "tools/list": result = {"tools": listed(self.bridge.screen)} elif method == "tools/call": @@ -846,7 +1018,7 @@ def reply(call, value, image, error): wire.opened() call = Call(name, params.get("arguments", {}), reply) try: - checked(call, self.bridge.screen) + checked(call, self.bridge.screen, self.bridge.folder) if name == "info": call.answer(self.bridge.info()) return @@ -877,9 +1049,10 @@ def ready(path, **fields): def lifeline(fd, bridge=None): - """The read end of a pipe from the driver. A line TURN_END ends the agent's turn (``bridge.end_turn``); any other - line is ignored. Once the driver's end closes (the driver ended, was killed, or let the unit go), the runner stops - as on SIGTERM. Run it in a thread of its own.""" + """The read end of a pipe from the driver. A line TURN_END ends the agent's turn (``bridge.end_turn``); a line + TASK_GIVEN closes the window before the task (``bridge.close_window``); any other line is ignored. Once + the driver's end closes (the driver ended, was killed, or let the unit go), the runner stops as on SIGTERM. Run it + in a thread of its own.""" buf = b"" try: while True: @@ -891,6 +1064,8 @@ def lifeline(fd, bridge=None): line, _, buf = buf.partition(b"\n") if line.strip() == TURN_END and bridge is not None: bridge.end_turn() + elif line.strip() == TASK_GIVEN and bridge is not None: + bridge.close_window() buf = buf[-4096:] except OSError: pass diff --git a/app/nullius/standard/definitions.py b/app/nullius/standard/definitions.py index 76b84d7e..17b7f921 100644 --- a/app/nullius/standard/definitions.py +++ b/app/nullius/standard/definitions.py @@ -1327,6 +1327,8 @@ def problems(d: dict, kind: str) -> List[str]: bad.append("driver.participant: true or false - whether setup is told the participant's kind") if "unit_end" in drv and not isinstance(drv["unit_end"], bool): bad.append("driver.unit_end: true or false - whether the driver is told each unit's end") + if "task_given" in drv and not isinstance(drv["task_given"], bool): + bad.append("driver.task_given: true or false - whether the driver is told when the agent is given its task") bad += _mcp_servers(d.get("mcp_servers"), isinstance(drv, dict) and "install" in drv) ds = d.get("datasets") if not isinstance(ds, list) or not ds: diff --git a/app/nullius/standard/kits/agent-computer-v1/driver.py b/app/nullius/standard/kits/agent-computer-v1/driver.py index fb2bfef2..897d3172 100644 --- a/app/nullius/standard/kits/agent-computer-v1/driver.py +++ b/app/nullius/standard/kits/agent-computer-v1/driver.py @@ -1,15 +1,24 @@ -"""Kit for agent.computer/v1: eight checks on a small real desktop, decided by the desktop's own records. +"""Kit for agent.computer/v1: nine checks on a small real desktop, decided by the desktop's own records. Basic: ``see``, ``click``, ``type``, ``limit``. Skills: ``drag``, ``scroll``, ``recover``, ``batch`` -(``checks.py`` gives each one's screen and verdict). +(``checks.py`` gives each one's screen and verdict) and ``late``. **A unit.** ``setup`` starts a desktop of the kit's image (``desktop.py``) and the fixture on it, then, in this driver's process, the port's server over that desktop (``computer_port.Bridge`` and ``Server``, its socket in a folder of this user's alone) and the kit's loop: a thread that asks the bridge for a step, runs its actions with ``Desktop.act`` and waits the pause after each, as a desktop benchmark's own loop does. The session gets the -server through ``computer_pipe.py`` (``mcp_servers.computer``). ``score`` ends the turn, waits for the loop, -reads the fixture's log, the actions' times and the call log, judges, and removes the desktop and the socket. -``oracle`` is the reference: the right calls, sent to the unit's own server as a client. +server through ``computer_pipe.py`` (``mcp_servers.computer``). The server's window before the task is closed by +``task_given`` (which the definition's ``driver.task_given`` asks Nullius to send) or, for the reference, as it +starts; every prepare call is recorded in ``verifier/computer-prepare.jsonl``, the agent's folder is ``FOLDER``. +``score`` ends the turn, waits for the loop, reads the fixture's log, the actions' times and the call log, judges, +and removes the desktop and the socket. ``oracle`` is the reference: the right calls, sent to the unit's own server +as a client. + +**late.** One labelled button appears ``LATE_SECONDS`` after the task is given, away from the screen's centre at a +place drawn for the unit (``late_place``), so that no click at a fixed point finds it; the agent clicks it. It passes +when the click is the check ``click``'s, and a picture was taken as a step ended, after the button was up and before +the action holding the press began. Its times are the driver's own clock (when the button was up, +each picture of a step's end, each action's start), the press matched to its action on the desktop's clock. **What is left when a unit never reaches ``score``.** Nothing: a new ``setup`` ends the unit before it, and the end of the driver's input (the run is over, or stopped) ends the one still open. A driver that is killed cannot @@ -23,6 +32,7 @@ import importlib.util import json import os +import secrets import shutil import socket import subprocess @@ -49,10 +59,13 @@ def _load(name, file): checks = _load("nullius_kit_checks", "checks.py") fixture = _load("nullius_kit_fixture", "fixture.py") -UNITS = checks.CHECKS +UNITS = checks.CHECKS + ("late",) PAUSE = 1.0 # after each action -PYTHON_SECONDS = 90 # one python call's reply +PYTHON_SECONDS = 90 # one desktop call's reply AGENT_SECONDS = 600 # the agent's turn +FOLDER = "/tmp/agent" # the agent's own folder in the desktop +LATE_SECONDS = 3.0 # late: the button appears this long after the task is given +LATE_PLACES = ((0.15, 0.35), (0.65, 0.85)) # late: where the button's centre may be, as fractions of each side LOOP_SECONDS = PYTHON_SECONDS + desktop.SETTLE + PAUSE + 30 # the loop's end, once the turn has ended USE = "Use the tools of the MCP server named computer, given to this session: they show and drive a desktop. " DONE = " Then reply with the word done." @@ -78,6 +91,9 @@ def prompt(unit, expected): w = expected["words"] return (USE + "Fill the three text fields on the screen: " + ", ".join(f"{w[f]} in the field labelled {f}" for f in checks.FIELDS) + "." + DONE) + if unit == "late": + return (USE + f"A button labelled {expected['target']} appears on the screen a few seconds after you are " + f"given this task. Click it once." + DONE) raise ValueError(f"no unit {unit!r}") @@ -142,6 +158,21 @@ def password(self): return self.d.password() +class Pictures(KitDesktop): + """The unit's desktop, with each step's picture timed on the driver's clock.""" + + def __init__(self, d, unit): + super().__init__(d) + self.unit = unit + + def capture(self): + t = time.monotonic() + png = super().capture() + if self.unit.bridge.running is not None: + self.unit.pictures.append((t, time.monotonic())) + return png + + class KitBridge(port.Bridge): """The port's bridge, with the kit's budget count and first acting call's recover error.""" @@ -196,7 +227,7 @@ def reply(call, value, image, error): if self.bridge.booked >= self.bridge.steps: self.bridge.past += 1 try: - port.checked(call, self.bridge.screen) + port.checked(call, self.bridge.screen, self.bridge.folder) except port.Refused as e: call.answer(error=str(e)) return @@ -207,35 +238,83 @@ def reply(call, value, image, error): class Unit: - """One unit's desktop, server and loop.""" + """One unit's desktop, server and loop; ``late`` draws its button once the task is given.""" def __init__(self, unit, dir): self.unit, self.trial = unit, os.path.dirname(os.path.abspath(dir)) - self.values, self.expected = checks.draw(unit) + if unit == "late": # the click screen with one button, drawn later + label = secrets.SystemRandom().choice(checks.LABELS) + self.values = {"screen": "click", "labels": [label], "at": late_place()} + self.expected = {"target": label, "labels": [label]} + else: + self.values, self.expected = checks.draw(unit) self.steps = checks.steps(unit) self.log = os.path.join(self.trial, "verifier", "computer-calls.jsonl") + self.prepared = os.path.join(self.trial, "verifier", "computer-prepare.jsonl") self.desktop = self.bridge = self.server = self.loop = self.folder = self.boxes = None self.final, self.error = "", None + self.pictures, self.acted, self.shown = [], [], None + self.given, self.up, self.ending = threading.Event(), threading.Event(), threading.Event() def start(self, job): os.makedirs(os.path.dirname(self.log), exist_ok=True) - if os.path.exists(self.log): - os.remove(self.log) + for path in (self.log, self.prepared): + if os.path.exists(path): + os.remove(path) image = desktop.image(os.path.dirname(self.log)) name = f"nullius-desktop-{os.urandom(6).hex()}" REAPER.tell(f"+container {name}") # before it exists: a kill during the start is covered self.desktop = desktop.Desktop.start(image, name, job=job) REAPER.tell(f"=container {name}") # only a successful start rules out late creation - self.desktop.start_fixture(self.values) - self.boxes = self.desktop.events()[0].get("boxes") or {} + acting = self.desktop.act + + def act(code, seconds=PYTHON_SECONDS): # each action's start on the driver's clock too + t = time.monotonic() + r = acting(code, seconds) + if r.get("start") is not None: + self.acted.append((t, r["start"], r["end"])) + return r + self.desktop.act = act + if self.unit != "late": + self.desktop.start_fixture(self.values) + self.boxes = self.desktop.events()[0].get("boxes") or {} folder = os.path.join(tempfile.gettempdir(), f"nullius-computer-{os.urandom(6).hex()}") REAPER.tell(f"+folder {folder}") # named before it is made self.folder = folder os.mkdir(folder, 0o700) - self.bridge = KitBridge(KitDesktop(self.desktop), self.steps, PAUSE, desktop.SCREEN, PYTHON_SECONDS) + self.bridge = KitBridge(Pictures(self.desktop, self), self.steps, PAUSE, desktop.SCREEN, FOLDER, window=True, + prepared=self.prepared) self.server = KitServer(self.bridge, self.socket(), log=self.log) self.loop = threading.Thread(target=self.run_loop, name=f"kit-loop-{self.unit}", daemon=True) self.loop.start() + if self.unit == "late": + threading.Thread(target=self.late, name="kit-late", daemon=True).start() + + def task_given(self): + self.bridge.close_window() + self.given.set() + + def late(self): + """The button drawn ``LATE_SECONDS`` after the task is given, unless the unit ends first.""" + while not self.given.wait(0.5): + if self.ending.is_set(): + return + if self.ending.wait(LATE_SECONDS): + return + try: + self.desktop.start_fixture(self.values) + self.boxes = self.desktop.events()[0].get("boxes") or {} + self.shown = time.monotonic() + except Exception as e: # the unit says so at score + self.error = f"the button could not be drawn: {type(e).__name__}: {e}" + finally: + self.up.set() + + def events(self): + """The fixture's log; none for ``late`` while its button was never drawn.""" + if self.unit == "late" and self.shown is None and not self.boxes: + return [] + return self.desktop.events() def socket(self): return os.path.join(self.folder, "s") @@ -276,6 +355,7 @@ def run_loop(self): def end(self, wait=LOOP_SECONDS): """The turn's end, the loop's (``wait`` seconds at most), the server's: what the verdict reads is complete.""" + self.ending.set() if self.bridge is not None: self.bridge.end_turn() if self.loop is not None: @@ -365,6 +445,30 @@ def job_of(dir): return os.path.basename(os.path.dirname(os.path.dirname(os.path.abspath(dir)))) or None +def late_place(rng=None): + """Where ``late`` draws its button: the point its frame is centred on, as fractions of the screen's width and + height, each in one of ``LATE_PLACES``. The button never covers the screen's centre, and the place is drawn anew + for each unit.""" + rng = rng or secrets.SystemRandom() + return [round(rng.uniform(*rng.choice(LATE_PLACES)), 3) for _ in range(2)] + + +def late_verdict(u, events, actions, message): + """``late``: the click's verdict, and an observation of the button, taken as a step ended, before the press.""" + v = checks.click(u.expected, events, actions, message) + if not v["passed"]: + return v + if u.shown is None: + return {"passed": False, "why": "the button was never drawn"} + press = next(e for e in events if e.get("kind") == "press" and e.get("hit") == u.expected["target"] + and e.get("button") == 1) + sent = next((h for h, s, e in u.acted if s <= press["t"] <= e + checks.SLACK), None) + if sent is None or not any(start > u.shown and end < sent for start, end in u.pictures): + return {"passed": False, "why": "no step's observation showed the button before the press: it was pressed " + "without looking again by a step"} + return {"passed": True, "why": ""} + + def units(dataset, **_): return {"units": list(UNITS)} @@ -387,6 +491,16 @@ def setup(unit, dir, **_): "agent_seconds": AGENT_SECONDS} +def task_given(unit, dir, **_): + """The agent is being given its task: the window before it closes, and ``late``'s count starts.""" + with OPEN_LOCK: + u = OPEN.get(unit) + if u is None: + raise ValueError(f"{unit} was not set up") + u.task_given() + return {} + + def score(unit, dir, transcript, answer, **_): with OPEN_LOCK: u = OPEN.pop(unit, None) @@ -394,7 +508,7 @@ def score(unit, dir, transcript, answer, **_): raise ValueError(f"{unit} was not set up") try: u.end() - events = u.desktop.events() + events = u.events() actions = list(u.desktop.actions) calls = u.calls() try: @@ -404,7 +518,10 @@ def score(unit, dir, transcript, answer, **_): finally: u.remove() message = "\n".join(x for x in (answer or "", u.final) if x) - v = checks.verdict(unit, u.expected, events, actions, message, calls, u.bridge.past) + if unit == "late": + v = late_verdict(u, events, actions, message) + else: + v = checks.verdict(unit, u.expected, events, actions, message, calls, u.bridge.past) passed = v["passed"] and u.error is None detail = {"said": (answer or "")[:200], "steps": u.steps, "steps_used": u.bridge.used, "past_budget": u.bridge.past, "calls": [c.get("tool") for c in calls], "errors": sum(1 for c in calls if c.get("isError")), @@ -475,13 +592,18 @@ def read_words(png, scale, count): def oracle(unit, dir, **_): - """The reference: the calls a correct agent makes, sent to the unit's own server, through its desktop, to the - same scorer; the answer they lead to. Words are read from the screenshot the server gives, and a call that does - not answer as a correct agent's would (the one ``recover`` refuses aside) fails the reference.""" + """The reference: given its task at once, then the calls a correct agent makes, sent to the unit's own server, + through its desktop, to the same scorer; the answer they lead to. Words are read from the screenshot the server + gives, and a call that does not answer as a correct agent's would (the one ``recover`` refuses aside) fails the + reference. For ``late``, steps that look again until the button is up and an observation shows it, then its + click.""" with OPEN_LOCK: u = OPEN.get(unit) if u is None: raise ValueError(f"{unit} was not set up") + u.task_given() + if unit == "late": + return late_reference(u) e, b, answer = u.expected, u.boxes, "done" c = Client(u.socket()) @@ -530,6 +652,29 @@ def call(tool, arguments=None, failing=False): return {"answer": answer} +def late_reference(u): + """``late``'s reference: a look by a step (``wait``) until a step's observation was taken with the button up, + then the click.""" + c = Client(u.socket()) + try: + target = u.expected["target"] + for _ in range(u.steps - 1): + failed, _ = c.call("wait") + failed_shot, png = c.call("screenshot") + if failed or failed_shot or not isinstance(png, bytes): + raise ReferenceFailed("a look by a step did not answer as a correct agent's would") + if u.up.is_set() and u.pictures[-1][0] > (u.shown or float("inf")): + break # this step's observation was taken with the button up + else: + raise ReferenceFailed("the button was not seen within the steps") + failed, said = c.call("click", dict(zip("xy", centre(u.boxes[target])))) + if failed: + raise ReferenceFailed(f"click answered with an error: {str(said)[:200]}") + finally: + c.close() + return {"answer": "done"} + + def end(unit, dir, **_): """Release the desktop before this attempt is archived for a retry.""" close_all(shutdown=False) @@ -539,7 +684,7 @@ def end(unit, dir, **_): def main(): atexit.register(REAPER.close) atexit.register(close_all) # runs first: the units go, then the reaper is let go with nothing - serve({"units": units, "setup": setup, "score": score, "oracle": oracle, "end": end}) + serve({"units": units, "setup": setup, "task_given": task_given, "score": score, "oracle": oracle, "end": end}) if __name__ == "__main__": diff --git a/app/nullius/standard/kits/agent-computer-v1/fixture.py b/app/nullius/standard/kits/agent-computer-v1/fixture.py index b5c80781..9b12b85c 100644 --- a/app/nullius/standard/kits/agent-computer-v1/fixture.py +++ b/app/nullius/standard/kits/agent-computer-v1/fixture.py @@ -4,7 +4,9 @@ It reads what to draw from the kit on its stdin, as one JSON object: - ``{"screen": "see", "words": [three words]}``: the words, drawn large; -- ``{"screen": "click", "labels": [six words]}``: six buttons, each labelled; +- ``{"screen": "click", "labels": [six words]}``: six buttons, each labelled, + in the screen's centre or, with ``"at": [x, y]``, around that point, given + as fractions of the screen's width and height; - ``{"screen": "type"}``: a text field; - ``{"screen": "drag"}``: a handle labelled MOVE and a zone labelled HERE; - ``{"screen": "scroll", "words": [two words]}``: a list with the words at @@ -219,7 +221,8 @@ def _see(self): def _click(self): frame = self.tk.Frame(self.root, background=BACKGROUND) - frame.place(relx=0.5, rely=0.5, anchor="center") + x, y = self.values.get("at") or (0.5, 0.5) + frame.place(relx=x, rely=y, anchor="center") labels = self.values["labels"] for i, label in enumerate(labels): img = bitmap(self.tk, frame, label, 5, paper="#ffffff") diff --git a/app/nullius/standard/kits/agent-computer-v1/standard.json b/app/nullius/standard/kits/agent-computer-v1/standard.json index 8062a9b5..d4bc04e8 100644 --- a/app/nullius/standard/kits/agent-computer-v1/standard.json +++ b/app/nullius/standard/kits/agent-computer-v1/standard.json @@ -1,9 +1,9 @@ { "schema": "nullius.benchmark/v1", "key": "agent-computer-v1", - "kit_version": 1, + "kit_version": 2, "label": "Kit: agent.computer/v1", - "summary": "Eight checks on a small real desktop, reached through the computer port's MCP server: four basic - read the screen, click, type, stop at the step budget - and four skills - drag, scroll, recover from an error, several actions in one step. The desktop's own records decide each one.", + "summary": "Nine checks on a small real desktop, reached through the computer port's MCP server, where the desktop is reached through steps alone once the task is given: four basic - read the screen, click, type, stop at the step budget - and five skills - drag, scroll, recover from an error, several actions in one step, and a button that appears late, seen by looking again with a step. The desktop's own records decide each one.", "family": "agent", "needs": [ "agent.computer/v1" @@ -18,10 +18,12 @@ "drag": "drags a handle into a marked zone", "scroll": "scrolls a list to read what it shows at first below its edge", "recover": "reads an acting call's error and acts again", - "batch": "sends several actions in one step" + "batch": "sends several actions in one step", + "late": "looks again with a step to find what appeared after the task was given" }, "driver": { "unit_end": true, + "task_given": true, "command": [ "python3", "{here}/driver.py" @@ -50,7 +52,7 @@ "datasets": [ { "id": "kit", - "units": 8 + "units": 9 } ], "units": { @@ -69,7 +71,7 @@ }, "score": { "kind": "pass", - "says": "the desktop's records show the task done, every pointer and key event inside an acting call, and the final message holds what the screen showed" + "says": "the desktop's records show the task done, every pointer and key event inside an acting call, and the final message holds what the screen showed; for late, a step's observation showed the button before the press" }, "limits": { "turn_seconds": 600 @@ -114,44 +116,49 @@ "label": "Several actions in one step", "task": "Fill three text fields with a budget of two steps.", "passes": "Each field holds its word, every key inside an acting call." + }, + "late": { + "label": "Look again by a step", + "task": "A button appears a few seconds after the task is given, away from the screen's centre at a place drawn for each unit; click it once.", + "passes": "The desktop logged a left press on the button with every input inside an acting call. A picture was taken at a step's end after the button appeared and before the action containing the press began." } }, "proven": { "kit": { "units": { - "units": 8, - "passed": "8 ids, the same twice, unique", - "on": "2026-11-16" + "units": 9, + "passed": "9 ids, the same twice, unique", + "on": "2026-11-19" }, "leak": { - "units": 8, - "looked_for": 2, - "short": 6, - "passed": "8/8", - "note": "6 reference(s) under 12 characters not looked for: too short to tell a leak from chance", - "on": "2026-11-16" + "units": 9, + "looked_for": 1, + "short": 8, + "passed": "9/9", + "note": "8 reference(s) under 12 characters not looked for: too short to tell a leak from chance", + "on": "2026-11-19" }, "nop": { - "passed": "0/8", + "passed": "0/9", "value": 0.0, "expect": { "at_most": 0.0 }, - "units": 8, - "job": "2026-11-16__13-07-27__agent-computer-v1__kit__prove-nop", + "units": 9, + "job": "2026-11-19__10-32-52__agent-computer-v1__kit__prove-nop", "source": "nullius", - "on": "2026-11-16" + "on": "2026-11-19" }, "oracle": { - "passed": "8/8", + "passed": "9/9", "value": 1.0, "expect": { "at_least": 1.0 }, - "units": 8, - "job": "2026-11-16__13-07-27__agent-computer-v1__kit__prove-oracle", + "units": 9, + "job": "2026-11-19__10-32-52__agent-computer-v1__kit__prove-oracle", "source": "nullius", - "on": "2026-11-16" + "on": "2026-11-19" } } } diff --git a/app/nullius/standard/ports.py b/app/nullius/standard/ports.py index 91d1785e..47341c94 100644 --- a/app/nullius/standard/ports.py +++ b/app/nullius/standard/ports.py @@ -55,7 +55,8 @@ "agent.computer/v1": { "label": "Computer use", "family": "agent", - "summary": "A desktop the agent sees in screenshots and drives with mouse and keyboard actions.", + "summary": "A desktop the agent sees in screenshots and drives with mouse and keyboard actions, reached through " + "steps alone: what the agent learns of it comes in a step's answer.", "kit": "agent-computer-v1", }, "cap.memory/v1": { diff --git a/app/nullius/standard/run.py b/app/nullius/standard/run.py index 9f253f43..4b17e9b3 100644 --- a/app/nullius/standard/run.py +++ b/app/nullius/standard/run.py @@ -129,6 +129,11 @@ class BenchmarkRefused(Refused): """This benchmark cannot run here; another benchmark may still run.""" +def computer_port(bench: dict) -> bool: + """Whether the benchmark needs the computer port.""" + return any(str(p).startswith("agent.computer/") for p in bench.get("needs") or []) + + def setup_request(bench: dict, kind: str, attempt: Optional[int] = None) -> tuple: """What a unit's ``setup`` is sent besides the unit and its folder, and how long it may take, for a unit that ``kind`` runs (agent, capability, control): ``attempt`` when the benchmark tells attempts apart @@ -537,7 +542,7 @@ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, datas raise Refused(f"backend {backend!r} is not one of {list(BACKENDS)}") if backend == "boat" and other.get("_kind") not in ("agent", "capability"): backend = "local" # a check or a setup: nothing of an agent's runs, so nothing goes to a VM - if backend == "boat" and other.get("_kind") == "agent" and "agent.computer/v1" in bench.get("needs", []): + if backend == "boat" and other.get("_kind") == "agent" and computer_port(bench): raise BenchmarkRefused("the computer port uses a socket on this machine, which an agent in a Boat VM " "cannot reach: choose local processes or the container backend") if backend == "boat": @@ -1290,6 +1295,18 @@ def unit_ended(self, unit: str, ws: str) -> bool: return True return False + def task_given(self, unit: str, ws: str) -> None: + """A driver that asks for it (``driver.task_given``) is told, and has answered, before the agent is sent its + task: its first prompt, or its command, which holds it. A computer port closes its window before the task + there. A failure ends the unit as its environment's, the task never sent.""" + if self.other.get("_kind") != "agent" or not (self.bench.get("driver") or {}).get("task_given"): + return + try: + self.driver().request("task_given", unit=unit, dir=ws, timeout=END_SECONDS) + except jsonl.Failed as e: + raise _Ended("EnvironmentSetupError", f"the benchmark was not told that the task is given: {e}", + retryable=not isinstance(e, jsonl.NoReply)) + def trial_name(self, i: int, unit: str, r: int = 1) -> str: return f"{_slug(unit)}__u{i:04d}" + (f"r{r}" if self.repeat > 1 else "") @@ -1312,7 +1329,7 @@ def unit(self, i: int, unit: str, r: int = 1, retry: int = 0) -> Optional[dict]: finally: asked, self.local.set_up = self.local.set_up, None if asked is not None: - if self.unit_ended(*asked) and "agent.computer/v1" in self.bench.get("needs", []): + if self.unit_ended(*asked) and computer_port(self.bench): # A failed end leaves the agent's desktop for the driver's exit cleanup. d = getattr(self.local, "driver", None) if d is not None: @@ -2268,6 +2285,8 @@ def left() -> float: opened = not first.get("done") if opened: prompt = prompt + "\n\n" + first["message"] + if opened: + self.task_given(unit, ws) # before the first prompt: the window from the session's start closes for _ in range(self.max_turns if opened else 0): transcript.append({"role": "user", "text": prompt}) try: @@ -2311,6 +2330,7 @@ def left() -> float: def _command_agent(self, a, env, ws, home, setup, transcript, log, tag, unit, rl, box=None): cmd = install.words(a["start"]["command"], {"here": a["_here"], "install": self.inst, "prompt": setup["prompt"]}) cmd, env = self.place(rl, cmd, env, ws, home, interactive=False, box=box) + self.task_given(unit, ws) # its command holds its task: its session starts with no window transcript.append({"role": "user", "text": setup["prompt"]}) t = time.time() try: diff --git a/app/tests/fixtures/osworld1_fake_desktop/desktop_env/desktop_env.py b/app/tests/fixtures/osworld1_fake_desktop/desktop_env/desktop_env.py index 0d38cb82..71076a3c 100644 --- a/app/tests/fixtures/osworld1_fake_desktop/desktop_env/desktop_env.py +++ b/app/tests/fixtures/osworld1_fake_desktop/desktop_env/desktop_env.py @@ -21,9 +21,11 @@ def __init__(self, env): def execute_python_command(self, command): first = command.splitlines()[0] if command else "" self.env.record("execute", first=first[:200]) + if command.rstrip().endswith("print('made')"): # the computer port making the agent's folder + return {"status": "success", "output": "made\n", "error": "", "returncode": 0} return {"status": "success", "output": f"{len(command)} {first}", "error": "", "returncode": 0} - def get_file(self, path): + def get_file(self, path, strict=False): self.env.record("get_file", path=path) return self.env.files.get(path) diff --git a/app/tests/fixtures/standard/computer_merge_driver.py b/app/tests/fixtures/standard/computer_merge_driver.py index d3461e0c..a293c817 100644 --- a/app/tests/fixtures/standard/computer_merge_driver.py +++ b/app/tests/fixtures/standard/computer_merge_driver.py @@ -14,6 +14,9 @@ class Desktop: + def python(self, code): + return {"status": "success", "output": "made\n", "error": "", "returncode": 0} + def capture(self): return b"fixture image" @@ -22,7 +25,7 @@ def password(self): def desktop(socket, log, inert=False): - bridge = port.Bridge(Desktop(), 2, 0, (1920, 1080), 1) + bridge = port.Bridge(Desktop(), 2, 0, (1920, 1080), "/tmp/agent") if inert: Path(socket).touch() server = port.Server.__new__(port.Server) diff --git a/app/tests/test_computer_kit_driver.py b/app/tests/test_computer_kit_driver.py index ab2da10f..b7b1d9b9 100644 --- a/app/tests/test_computer_kit_driver.py +++ b/app/tests/test_computer_kit_driver.py @@ -1,13 +1,14 @@ -"""The computer kit's eight checks and its driver: each verdict on hand-written records; the driver's wiring -(the port's server and the kit's loop over a desktop, the turn's end at score, the budget, the reference, a unit -let go without its score, the reaper) over a fake desktop, with no container; the docker tier runs the eight -checks on real desktops, the reference passing each and the null control none.""" +"""The computer kit's nine checks and its driver: each verdict on hand-written records; the driver's wiring +(the port's server and the kit's loop over a desktop, the window before the task, the turn's end at score, the +budget, the reference, ``late``, a unit let go without its score, the reaper) over a fake desktop, with no container; +the docker tier runs the nine checks on real desktops, the reference passing each and the null control none.""" from __future__ import annotations import importlib.util import inspect import io import json +import math import os import queue import random @@ -150,9 +151,9 @@ def test_the_port_names_the_kit_and_the_kit_its_checks(self): self.assertEqual(ports.PORTS["agent.computer/v1"]["kit"], "agent-computer-v1") self.assertEqual(kit["key"], "agent-computer-v1") self.assertEqual(definitions.problems({k: v for k, v in kit.items() if not k.startswith("_")}, "benchmark"), []) - self.assertEqual(list(kit["check_details"]), list(checks.CHECKS)) + self.assertEqual(list(kit["check_details"]), list(driver.UNITS)) self.assertEqual(kit["basic"], list(checks.BASIC)) - self.assertEqual(set(kit["skills"]), set(checks.CHECKS) - set(checks.BASIC)) + self.assertEqual(set(kit["skills"]), set(driver.UNITS) - set(checks.BASIC)) self.assertEqual(kit["environments"]["desktop"], dict(desktop.KIT["environments"]["desktop"], summary=kit["environments"]["desktop"]["summary"])) self.assertEqual(kit["mcp_servers"]["computer"]["command"], ["python3", "{here}/pipe.py"]) # a file of its folder @@ -163,7 +164,7 @@ def test_the_port_names_the_kit_and_the_kit_its_checks(self): def test_the_pipe_from_the_kits_folder_joins_stdio_to_the_socket(self): with tempfile.TemporaryDirectory(dir=os.environ.get("TMPDIR")) as tmp: path = os.path.join(tmp, "s") - server = port.Server(port.Bridge(FakeDesktop("d", {}), 1, 0, (1920, 1080), 1), path) + server = port.Server(port.Bridge(FakeDesktop("d", {}), 1, 0, (1920, 1080), driver.FOLDER), path) try: p = subprocess.run([sys.executable, os.path.join(KIT, "pipe.py"), path], capture_output=True, timeout=30, input=(json.dumps({"jsonrpc": "2.0", "id": 1, "method": "tools/list"}) + "\n").encode()) @@ -172,11 +173,21 @@ def test_the_pipe_from_the_kits_folder_joins_stdio_to_the_socket(self): self.assertEqual(p.returncode, 0, p.stderr) self.assertEqual(json.loads(p.stdout)["result"]["tools"][0]["name"], "info") + def test_the_kit_has_nine_checks_and_is_told_the_task(self): + kit = definitions.kit("agent.computer/v1") + self.assertEqual((kit["key"], kit["needs"], kit["datasets"]), ("agent-computer-v1", ["agent.computer/v1"], + [{"id": "kit", "units": 9}])) + self.assertTrue(kit["driver"]["task_given"] and kit["driver"]["unit_end"]) + self.assertEqual(driver.units("kit")["units"], list(checks.CHECKS) + ["late"]) + self.assertEqual(set(kit["check_details"]), set(driver.UNITS)) + self.assertEqual(set(kit["basic"]) | set(kit["skills"]), set(driver.UNITS)) + # -- the driver over a fake desktop ------------------------------------------------------------------------------------- class FakeDesktop: """A desktop with no container: the fixture's boxes, and the input each click, write or key of an action (or - of a python call) would send, logged at the time it runs, as the fixture would.""" + of the desktop's python) would send, logged at the time it runs, as the fixture would. The agent's folder is made + as the bridge asks.""" made = [] def __init__(self, name, values): @@ -211,6 +222,8 @@ def act(self, code, seconds=None): return {"start": start, "end": self.actions[-1]["end"], "reply": {"status": "success", "returncode": 0}} def python(self, code, seconds=None): + if code == port.MAKE_FOLDER.format(folder=driver.FOLDER): + return {"status": "success", "output": "made\n", "error": "", "returncode": 0} self._input(code) return {"status": "success", "output": "", "error": "", "returncode": 0} @@ -254,7 +267,7 @@ def unit(self): u.expected = {"words": ["ACORN", "BASKET", "CANDLE"]} d = u.desktop = FakeDesktop("fake", u.values) u.boxes = d.boxes - u.bridge = driver.KitBridge(driver.KitDesktop(d), u.steps, 0, desktop.SCREEN, 1) + u.bridge = driver.KitBridge(driver.KitDesktop(d), u.steps, 0, desktop.SCREEN, driver.FOLDER) driver.OPEN["see"] = u return ws, u, d @@ -377,10 +390,14 @@ def draw(check, rng=None): self.enterContext(mock.patch.object(desktop, "_CLIENTS_CLOSED", False)) self.addCleanup(driver.close_all) - def unit(self, name): + def unit(self, name, given=True): + """A unit set up, and given its task (the window before it closed) unless ``given`` is false.""" ws = os.path.join(self.tmp, "job", f"{name}-{len(FakeDesktop.made)}", "workspace") os.makedirs(ws) - return ws, driver.setup(name, ws) + s = driver.setup(name, ws) + if given: + driver.task_given(name, ws) + return ws, s class Wiring(FakeUnits): @@ -420,7 +437,7 @@ def test_the_units_record_keeps_the_fixtures_events_and_where_its_targets_were(s self.assertEqual(json.load(f), u.boxes) def test_the_turns_end_at_score_ends_the_loop_and_judges_what_is_there(self): - ws, s = self.unit("click") + ws, s = self.unit("click", given=False) u = driver.OPEN["click"] t = time.monotonic() got = driver.score("click", ws, [], "") # no call: the null control @@ -449,16 +466,6 @@ def test_a_step_with_an_end_is_one_step_and_ends_the_kits_loop(self): self.assertEqual((scored["detail"]["steps_used"], scored["detail"]["actions"]), (1, 3)) self.assertEqual(u.final, "typed") - def test_a_client_that_types_through_python_fails(self): - ws, s = self.unit("type") - u = driver.OPEN["type"] - c = driver.Client(s["mcp"][0]["args"][0]) - c.call("python", {"code": f"import pyautogui; pyautogui.write({u.expected['word']!r}); pyautogui.hotkey('enter')"}) - c.close() - got = driver.score("type", ws, [], "done") - self.assertFalse(got["passed"]) - self.assertIn("outside every action", got["detail"]["why"]) - def test_a_client_that_acts_past_its_budget_fails(self): ws, s = self.unit("limit") u = driver.OPEN["limit"] @@ -618,7 +625,7 @@ def unit(self, name, steps=2): u.desktop = FakeDesktop("fake", {"labels": ["OCEAN"]}) u.expected = {"target": "OCEAN", "labels": ["OCEAN"]} u.steps = steps - u.bridge = driver.KitBridge(driver.KitDesktop(u.desktop), steps, 0, desktop.SCREEN, 1) + u.bridge = driver.KitBridge(driver.KitDesktop(u.desktop), steps, 0, desktop.SCREEN, driver.FOLDER) s = driver.KitServer.__new__(driver.KitServer) s._state(u.bridge, "/unused/socket", None) calls = [] @@ -632,13 +639,13 @@ def test_an_invalid_call_admitted_before_exhaustion_is_not_counted_past_the_budg entered, release = threading.Event(), threading.Event() original, seen, wires = port.checked, [], [] - def checking(call, screen): + def checking(call, screen, folder): if call.args.get("x") == "bad": seen.append(call) entered.set() if not release.wait(5): raise RuntimeError("validation was not released") - return original(call, screen) + return original(call, screen, folder) t = threading.Thread(target=lambda: wires.append(send(s, "click", {"x": "bad", "y": 150})), daemon=True) with mock.patch.object(port, "checked", side_effect=checking): @@ -669,12 +676,12 @@ def test_validation_cannot_move_a_calls_start_past_the_recovery_error(self): first = send(s, "click", {"x": 150, "y": 150}) _, actions = u.bridge.predict("", None) - def checking(call, screen): + def checking(call, screen, folder): checked.append(call) entered.set() if not release.wait(5): raise RuntimeError("validation was not released") - return original(call, screen) + return original(call, screen, folder) t = threading.Thread(target=lambda: wires.append(send(s, "click", {"x": 150, "y": 150})), daemon=True) with mock.patch.object(port, "checked", side_effect=checking): @@ -831,22 +838,23 @@ def act(code, seconds=None): self.assertEqual(got["detail"]["steps_used"], 2) def test_the_reference_reads_the_words_from_the_screenshot_and_fails_without_one(self): - ws, s = self.unit("see") - fake, text = FakeDesktop.made[-1], " ".join(driver.OPEN["see"].expected["words"]) - fake.capture = lambda: _png(text, driver.fixture.SEE_SCALE) - self.assertEqual(driver.oracle("see", ws)["answer"], text) + drawn = lambda fake: _png(" ".join(fake.values["words"]), driver.fixture.SEE_SCALE) + with mock.patch.object(FakeDesktop, "capture", drawn): # the start picture, taken as the unit starts + ws, s = self.unit("see") + text = " ".join(driver.OPEN["see"].expected["words"]) + self.assertEqual(driver.oracle("see", ws)["answer"], text) self.assertTrue(driver.score("see", ws, [], text)["passed"]) - ws, s = self.unit("see") - def broken(): + def broken(fake): raise desktop.DesktopError("the capture failed") - FakeDesktop.made[-1].capture = broken - with self.assertRaises(driver.ReferenceFailed): - driver.oracle("see", ws) - ws, s = self.unit("see") - FakeDesktop.made[-1].capture = lambda: _png("ACORN", 9) # a screen with only one word - with self.assertRaises(driver.ReferenceFailed): - driver.oracle("see", ws) + with mock.patch.object(FakeDesktop, "capture", broken): + ws, s = self.unit("see") + with self.assertRaises(driver.ReferenceFailed): + driver.oracle("see", ws) + with mock.patch.object(FakeDesktop, "capture", lambda fake: _png("ACORN", 9)): # a screen with only one word + ws, s = self.unit("see") + with self.assertRaises(driver.ReferenceFailed): + driver.oracle("see", ws) class Shutdown(unittest.TestCase): @@ -1125,7 +1133,218 @@ def test_the_driver_tells_the_reaper_before_the_container_exists(self): self.assertEqual(driver.OPEN, {}) -# -- the docker tier: the eight checks on real desktops -------------------------------------------------------------- +class Fake(FakeDesktop): + """A blank desktop that draws the fixture on request.""" + + def __init__(self, name, values): + super().__init__(name, {}) + self.boxes.clear() + self.log.clear() + + def start_fixture(self, values): + self.values = values + self.boxes = {label: [100 + 300 * i, 100, 300 + 300 * i, 200] + for i, label in enumerate(values.get("labels") or [])} + if values.get("at"): # one button, around the point the values name + x, y = int(values["at"][0] * desktop.SCREEN[0]), int(values["at"][1] * desktop.SCREEN[1]) + self.boxes = {label: [x - 120, y - 45, x + 120, y + 45] for label in values["labels"]} + self.boxes.update(field=[600, 500, 1200, 560], handle=[200, 500, 260, 560], zone=[1300, 400, 1700, 700], + list=[600, 300, 1300, 800]) + self.log.append({"kind": "ready", "t": time.monotonic(), "boxes": dict(self.boxes)}) + + def capture(self): + return super().capture() + json.dumps(self.boxes).encode() + + +class Fixture(unittest.TestCase): + def test_click_places_the_button_at_each_supplied_position(self): + fixture = driver.fixture + for at in ([0.2, 0.7], [0.8, 0.3]): + with self.subTest(at=at): + f = fixture.Fixture.__new__(fixture.Fixture) + f.tk, f.root = mock.Mock(), object() + f.values = {"labels": ["BUTTON"], "at": at} + f.images, f.names = [], {} + with mock.patch.object(fixture, "bitmap"): + f._click() + f.tk.Frame.return_value.place.assert_called_once_with(relx=at[0], rely=at[1], anchor="center") + + def test_the_fake_starts_blank_and_draws_the_button_only_when_the_fixture_starts(self): + values = {"screen": "click", "labels": ["BUTTON"]} + d = Fake("desktop", values) + self.assertEqual(d.boxes, {}) + self.assertEqual(d.events(), []) + before = d.capture() + self.assertEqual(d.hit(200, 150), "background") + d.start_fixture(values) + self.assertEqual(d.hit(200, 150), "BUTTON") + self.assertNotEqual(d.capture(), before) + self.assertEqual(len(d.events()), 1) + self.assertEqual(d.events()[0]["kind"], "ready") + self.assertEqual(d.events()[0]["boxes"], d.boxes) + + +def blind_clicks(sock, u): + """A control that must fail ``late``: a script that never looks. Once the task is given, one step clicks the + screen's centre and waits there until the button has had time to appear, so that the step's observation, which it + never reads, is taken with the button up; a second step clicks the centre again. A button drawn at the centre + would take that second click, after a look by a step.""" + c = driver.Client(sock) + try: + x, y = desktop.SCREEN[0] // 2, desktop.SCREEN[1] // 2 + waits = math.ceil((driver.LATE_SECONDS + 1) / driver.PAUSE) + 1 + said = [c.call("step", {"actions": [f"pyautogui.click({x}, {y})"] + ["WAIT"] * waits})] + if not u.up.wait(30): + raise AssertionError("the button was never drawn") + said.append(c.call("click", {"x": x, "y": y})) + finally: + c.close() + if any(failed for failed, _ in said): + raise AssertionError(f"a blind click was refused: {said}") + + +class Window(unittest.TestCase): + """Units over blank fake desktops, given their task only when the test says so.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp(dir=os.environ.get("TMPDIR")) + self.addCleanup(shutil.rmtree, self.tmp, True) + + def start(image, name, *, job=None, seconds=None): + return Fake(name, list(driver.OPEN.values())[-1].values) + self.enterContext(mock.patch.object(desktop, "image", return_value="img")) + self.enterContext(mock.patch.object(desktop.Desktop, "start", side_effect=start)) + self.enterContext(mock.patch.object(driver.REAPER, "tell")) + self.enterContext(mock.patch.object(driver, "PAUSE", 0.01)) + self.enterContext(mock.patch.object(driver, "LATE_SECONDS", 0.05)) + self.enterContext(mock.patch.object(desktop, "_CLIENTS_CLOSED", False)) + self.addCleanup(driver.close_all) + + def unit(self, name): + ws = os.path.join(self.tmp, "job", f"{name}-{len(FakeDesktop.made)}", "workspace") + os.makedirs(ws) + return ws, driver.setup(name, ws) + + def client(self, s): + c = driver.Client(s["mcp"][0]["args"][0]) + self.addCleanup(c.close) + return c + + def test_the_server_has_the_window_before_the_task(self): + ws, s = self.unit("click") + u, c = driver.OPEN["click"], self.client(s) + names = [t["name"] for t in c.ask("tools/list", {})["result"]["tools"]] + self.assertNotIn("python", names) + self.assertIn("prepare", names) + x, y = driver.centre(u.boxes[u.expected["target"]]) + failed, said = c.call("click", {"x": x, "y": y}) + self.assertTrue(failed) + self.assertIn("the task has not been given yet", said) + self.assertEqual(c.call("prepare", {"actions": ["print(1)"], "files": ["/tmp/agent/a"]}), + (False, {"files": {"/tmp/agent/a": None}})) + self.assertEqual(driver.task_given("click", ws), {}) + self.assertEqual(c.call("step", {"actions": [f"pyautogui.click({x}, {y})"], "files": ["/tmp/agent/a"]}), + (False, {"steps_left": u.steps - 1, "done": False, "files": {"/tmp/agent/a": None}})) + self.assertTrue(c.call("prepare", {"actions": []})[0]) + got = driver.score("click", ws, [], "done") + self.assertTrue(got["passed"], got) + self.assertEqual(got["detail"]["calls"], ["click", "prepare", "step", "prepare"]) + with open(os.path.join(os.path.dirname(ws), "verifier", "computer-prepare.jsonl")) as f: + self.assertEqual([json.loads(line)["actions"] for line in f], [["print(1)"]]) + + def test_input_sent_by_prepare_fails_the_check(self): + ws, s = self.unit("click") + u, c = driver.OPEN["click"], self.client(s) + x, y = driver.centre(u.boxes[u.expected["target"]]) + c.call("prepare", {"actions": [f"pyautogui.click({x}, {y})"]}) + driver.task_given("click", ws) + got = driver.score("click", ws, [], "done") + self.assertFalse(got["passed"]) + self.assertIn("outside every action", got["detail"]["why"]) + + def test_the_reference_is_given_its_task_as_it_starts(self): + for unit in ("click", "type", "limit", "recover"): # the fake desktop logs no field values + with self.subTest(unit=unit): + ws, s = self.unit(unit) + got = driver.score(unit, ws, [], driver.oracle(unit, ws)["answer"]) + self.assertTrue(got["passed"], got) + + def test_late_passes_for_the_reference_which_looks_again_by_steps(self): + ws, s = self.unit("late") + self.assertIn("appears on the screen a few seconds after", s["prompt"]) + u = driver.OPEN["late"] + self.assertEqual(u.desktop.boxes, {}) + self.assertEqual(u.desktop.events(), []) + got = driver.score("late", ws, [], driver.oracle("late", ws)["answer"]) + self.assertTrue(got["passed"], got) + self.assertIn("wait", got["detail"]["calls"]) + + def test_late_fails_a_click_sent_with_no_look_by_a_step_once_the_button_is_up(self): + ws, s = self.unit("late") + u, c = driver.OPEN["late"], self.client(s) + driver.task_given("late", ws) + self.assertTrue(u.up.wait(5)) + c.call("screenshot") # the start picture: no button + self.assertFalse(c.call("click", dict(zip("xy", driver.centre(u.boxes[u.expected["target"]]))))[0]) + got = driver.score("late", ws, [], "done") + self.assertFalse(got["passed"]) + self.assertIn("without looking again by a step", got["detail"]["why"]) + + def test_late_draws_its_button_away_from_the_centre_at_a_place_drawn_for_each_unit(self): + rng, places = random.Random(7), set() + for _ in range(1000): + place = tuple(driver.late_place(rng)) + for f, side in zip(place, desktop.SCREEN): + self.assertTrue(0.15 <= f <= 0.35 or 0.65 <= f <= 0.85, place) + self.assertGreater(abs(f - 0.5) * side, 0.15 * side - 1) # the centre is never on the button + places.add(place) + self.assertGreater(len(places), 990) + ws, s = self.unit("late") + first = driver.OPEN["late"].values["at"] + ws, s = self.unit("late") + self.assertNotEqual(driver.OPEN["late"].values["at"], first) + + def test_late_fails_blind_clicks_at_the_screens_centre(self): + ws, s = self.unit("late") + u = driver.OPEN["late"] + driver.task_given("late", ws) + blind_clicks(s["mcp"][0]["args"][0], u) + got = driver.score("late", ws, [], "done") + self.assertFalse(got["passed"]) + self.assertEqual(got["detail"]["steps_used"], 2) + self.assertIn(f"no left press on the button {u.expected['target']}", got["detail"]["why"]) + + def test_late_fails_the_null_control_and_draws_no_button_when_the_task_is_never_given(self): + ws, s = self.unit("late") + u = driver.OPEN["late"] + t = time.monotonic() + got = driver.score("late", ws, [], "") + self.assertLess(time.monotonic() - t, 5) + self.assertFalse(got["passed"]) + self.assertIsNone(u.shown) + + def test_late_fails_without_a_prior_step_when_the_start_picture_is_delayed(self): + original = Fake.python + + def delayed(d, code, seconds=None): + if "os.makedirs(p, 0o700)" in code: + self.assertTrue(driver.OPEN["late"].up.wait(5)) + return original(d, code, seconds) + + with mock.patch.object(Fake, "python", delayed): + ws, s = self.unit("late") + u, c = driver.OPEN["late"], self.client(s) + driver.task_given("late", ws) + self.assertFalse(c.call("screenshot")[0]) + self.assertEqual(u.bridge.used, 0) + self.assertFalse(c.call("click", dict(zip("xy", driver.centre(u.boxes[u.expected["target"]]))))[0]) + got = driver.score("late", ws, [], "done") + self.assertEqual(got["detail"]["steps_used"], 1) + self.assertFalse(got["passed"]) + self.assertIn("without looking again by a step", got["detail"]["why"]) + + +# -- the docker tier: the nine checks on real desktops -------------------------------------------------------------- MARGIN = 30 # process starts, protocol messages and scheduling @@ -1247,7 +1466,7 @@ def unit(self, name): return ws, driver.setup(name, ws) def test_the_reference_passes_every_check_and_the_null_control_none(self): - for unit in checks.CHECKS: + for unit in driver.UNITS: with self.subTest(unit=unit, control="oracle"): ws, _ = self.unit(unit) answer = driver.oracle(unit, ws)["answer"] @@ -1257,18 +1476,29 @@ def test_the_reference_passes_every_check_and_the_null_control_none(self): ws, _ = self.unit(unit) self.assertFalse(driver.score(unit, ws, [], "")["passed"]) - def test_scripted_clients_that_type_through_python_or_act_past_their_budget_fail(self): + def test_blind_clicks_at_the_screens_centre_fail_late(self): + ws, s = self.unit("late") + u = driver.OPEN["late"] + driver.task_given("late", ws) + blind_clicks(s["mcp"][0]["args"][0], u) + got = driver.score("late", ws, [], "done") + self.assertFalse(got["passed"], got) + self.assertIn("no left press on the button", got["detail"]["why"]) + + def test_scripted_clients_that_type_through_prepare_or_act_past_their_budget_fail(self): ws, s = self.unit("type") u = driver.OPEN["type"] c = driver.Client(s["mcp"][0]["args"][0]) x, y = driver.centre(u.boxes["field"]) - c.call("python", {"code": f"import pyautogui\npyautogui.click({x}, {y})\npyautogui.write({u.expected['word']!r})\n" - "pyautogui.press('enter')"}) + c.call("prepare", {"actions": [f"pyautogui.click({x}, {y})", f"pyautogui.write({u.expected['word']!r})", + "pyautogui.press('enter')"]}) c.close() + driver.task_given("type", ws) got = driver.score("type", ws, [], "done") self.assertFalse(got["passed"]) self.assertIn("outside every action", got["detail"]["why"]) ws, s = self.unit("limit") + driver.task_given("limit", ws) u = driver.OPEN["limit"] c = driver.Client(s["mcp"][0]["args"][0]) for label in checks.ORDER[:3]: @@ -1332,6 +1562,8 @@ def test_a_session_of_the_scripted_acp_agent_reaches_the_server_through_the_pipe {"jsonrpc": "2.0", "id": 2, "method": "session/new", "params": {"cwd": ws, "mcpServers": [server]}}, {"jsonrpc": "2.0", "id": 3, "method": "session/prompt", "params": {"sessionId": "s1", "prompt": [{"type": "text", "text": s["prompt"]}]}}): + if m["method"] == "session/prompt": + driver.task_given("click", ws) # as the run tells the driver before the task p.stdin.write(json.dumps(m) + "\n") p.stdin.flush() seconds = (2 * desktop.CAPTURE_SECONDS + driver.PYTHON_SECONDS + desktop.SETTLE diff --git a/app/tests/test_osworld1_agent_side.py b/app/tests/test_osworld1_agent_side.py index 1c5551a6..f8db11a1 100644 --- a/app/tests/test_osworld1_agent_side.py +++ b/app/tests/test_osworld1_agent_side.py @@ -179,6 +179,7 @@ def session(self, setup: dict) -> Session: s = Session(setup, self.ws) self.sessions.append(s) s.ask("initialize", {"protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "t"}}) + self.assertEqual(self.drivers[-1].request("task_given", unit=UNIT, dir=self.ws), {}) # as the run does return s def events(self, kind: str) -> list: diff --git a/app/tests/test_osworld1_entry_parts.py b/app/tests/test_osworld1_entry_parts.py index 2a90bb14..a0a99d0b 100644 --- a/app/tests/test_osworld1_entry_parts.py +++ b/app/tests/test_osworld1_entry_parts.py @@ -640,8 +640,9 @@ def __init__(self, close_seconds=0.0): self.seen, self.files, self.close_seconds = [], {}, close_seconds self.client_password = "password" self.controller = types.SimpleNamespace( - execute_python_command=lambda code: self.seen.append(("python", code)) or {"status": "success"}, - get_file=lambda path: self.files.get(path), + execute_python_command=lambda code: self.seen.append(("python", code)) or ( + {"status": "success", "output": "made\n"} if "os.makedirs(p, 0o700)" in code else {"status": "success"}), + get_file=lambda path, strict=False: self.files.get(path), get_screenshot=lambda: b"png") self.setup_controller = types.SimpleNamespace(_upload_file_setup=self.upload) @@ -686,7 +687,7 @@ def args(self): return types.SimpleNamespace(example=example, result_dir=os.path.join(self.tmp, "results"), socket=os.path.join(folder, "s"), ready=os.path.join(folder, "ready.json"), vm="vm", calls=os.path.join(self.tmp, "calls.jsonl"), held=None, lifeline=-1, - steps=3, pause=0.0) + steps=3, pause=0.0, prepared=None, given=None) def run_with(self, env, loop): a = self.args() @@ -740,6 +741,100 @@ def loop(agent, env_, example, steps, instruction, args, where, scores): self.assertEqual(code, 1) self.assertLess(time.monotonic() - t0, 3) + def test_the_entry_serves_the_computer_port_and_a_steps_observation_is_the_one_the_loop_hands(self): + from tests.test_standard_computer_port import Client, error, image, value + with open(os.path.join(ENTRY, "standard.json")) as f: + definition = json.load(f) + self.assertEqual(definition["needs"], ["agent.computer/v1"]) + self.assertTrue(definition["driver"]["task_given"]) + env, shots, said = FakeEnv(), iter([b"after-1", b"after-2"]), {} + env.step = lambda action, pause=2: ({"screenshot": next(shots)}, 0, False, {}) + env.controller.get_screenshot = lambda: self.fail("a capture was added to the loop's observation") + a = self.args() + a.given, a.steps = os.path.join(os.path.dirname(a.socket), "given.json"), 2 + turn = self.r.Turn() + turn.given, turn.said = a.given, computer_port.ready + + def agent(): # the agent's calls, while the loop waits in predict + c = Client(a.socket) + try: + c.initialize() + said["tools"] = [t["name"] for t in c.ask("tools/list")["result"]["tools"]] + said["start"] = image(c.call("screenshot")) + said["prepare"] = value(c.call("prepare", actions=["print(1)"])) + said["early"] = error(c.call("click", x=1, y=1)) + turn.close_window() + said["late"] = error(c.call("upload", path="/tmp/agent/a", data="YQ==")) + said["click"] = image(c.call("click", x=1, y=1)) + said["again"] = image(c.call("screenshot")) + said["step"] = value(c.call("step", actions=[], files=["/tmp/agent/none"])) + finally: + c.close() + + def loop(agent_, env_, example, steps, instruction, args, where, scores): + self.assertIsInstance(env_, self.r.Watched) + agent_.reset(None, vm_ip="x") + obs = {"screenshot": b"start"} + worker = threading.Thread(target=agent, daemon=True) + worker.start() + for _ in range(steps): + _, actions = agent_.predict(instruction, obs) + for action in actions: + obs = env_.step(action, 0)[0] + worker.join(10) + open(os.path.join(where, "result.txt"), "w").write("0.0\n") + + with mock.patch.dict(sys.modules, {"lib_run_single": types.SimpleNamespace(run_single_example=loop), + "desktop_env": mock.Mock(), + "desktop_env.desktop_env": types.SimpleNamespace(DesktopEnv=lambda **kw: env)}): + self.assertEqual(self.r.run(a, computer_port, computer_port.Stop(30), turn), 0) + self.assertNotIn("python", said["tools"]) + self.assertEqual((said["start"], said["prepare"]), (b"start", {"files": {}})) + self.assertIn("the task has not been given yet", said["early"]) + self.assertEqual(said["late"], "the task has been given: upload is taken only before it") + self.assertEqual((said["click"], said["again"]), (b"after-1", b"after-1")) + self.assertEqual(said["step"], {"steps_left": 0, "done": True, "files": {"/tmp/agent/none": None}}) + with open(a.given) as f: + self.assertEqual(json.load(f), {"ok": True}) + self.assertIn(("python", computer_port.PREPARE.format(code="print(1)")), env.seen) + + +class TaskGiven(unittest.TestCase): + """task_given: the runner is told on its lifeline and the driver answers once the runner has said so.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + self.d = load("driver", self.tmp) + self.ws = os.path.join(self.tmp, "trial", "workspace") + os.makedirs(self.ws) + + def runner(self, code): + read, write = os.pipe() + sock_dir = tempfile.mkdtemp(dir=self.tmp) + p = subprocess.Popen([sys.executable, "-c", code, str(read), sock_dir], pass_fds=(read,), start_new_session=True, + stdin=subprocess.DEVNULL) + os.close(read) + self.addCleanup(lambda: (p.poll() is None and p.kill(), p.wait(), os.close(write))) + self.d.RUNNERS[self.d._runner_dir(self.ws)] = {"p": p, "sock_dir": sock_dir, "life": write} + return sock_dir + + def test_the_driver_answers_once_the_runner_says_the_window_is_closed(self): + sock_dir = self.runner("import os, sys, json, time\nfd, data = int(sys.argv[1]), b''\n" + "while b'task-given\\n' not in data:\n data += os.read(fd, 64)\n" + "json.dump({'ok': True}, open(os.path.join(sys.argv[2], 'given.json'), 'w'))\n" + "time.sleep(30)\n") + self.assertEqual(self.d.task_given("os__t1", self.ws), {}) + self.assertFalse(os.path.exists(os.path.join(sock_dir, "given.json"))) + + def test_a_runner_that_never_says_so_or_has_none_fails_it(self): + self.d.GIVEN_SECONDS = 0.5 + with self.assertRaisesRegex(RuntimeError, "no runner of this driver's"): + self.d.task_given("os__t1", self.ws) + self.runner("import time; time.sleep(30)") + with self.assertRaisesRegex(RuntimeError, "did not say that the window before the task is closed"): + self.d.task_given("os__t1", self.ws) + if __name__ == "__main__": unittest.main() diff --git a/app/tests/test_site.py b/app/tests/test_site.py index b0e12198..5471ac8d 100644 --- a/app/tests/test_site.py +++ b/app/tests/test_site.py @@ -16,6 +16,14 @@ def module(name): class Website(unittest.TestCase): + def test_computer_port_states_the_desktop_code_and_named_path_limits(self): + page = (SITE.parents[1] / "docs/ports/agent-computer-v1.md").read_text() + limits = page.split("## Known limits\n", 1)[1].split("\n## ", 1)[0] + self.assertIn("Inside the desktop, the agent's code can read and write wherever the desktop's user can " + "and can create links outside `folder`; `folder` bounds the names of paths passed to " + "`upload` and `files`, and the benchmark's records hold the code of every step action " + "and `prepare` call.", limits) + def test_computer_port_links_to_the_step_option_without_repeating_its_default(self): page = (SITE.parents[1] / "docs/ports/agent-computer-v1.md").read_text() row = next(line for line in page.splitlines() if line.startswith("| The step budget |")) diff --git a/app/tests/test_standard_computer_port.py b/app/tests/test_standard_computer_port.py index 4aa9b98d..31aec25b 100644 --- a/app/tests/test_standard_computer_port.py +++ b/app/tests/test_standard_computer_port.py @@ -1,7 +1,10 @@ """The computer port's shared code (``nullius.standard.computer_port``): the MCP server ``computer`` and its bridge to a benchmark's loop, on a fake desktop, with a scripted MCP client over the server's socket; the runner's ready file, -lifeline and one stop. A loop of a few lines stands in for an upstream's, counting steps as OSWorld's -run_single_example does. The class UpstreamLoop drives OSWorld's own run_single_example with the bridge, where a +lifeline and one stop. Once the agent is given its task, the desktop is reached through acting calls alone: one +observation for each step, fixed as the step ends, which screenshot returns without a new picture; upload and prepare +in the window before the task alone, inside the agent's folder; a step's files. A loop of a few lines stands in for an +upstream's, counting steps as OSWorld's run_single_example does, handing a constant observation, its own (as OSWorld's) +or none (as the kit's). The class UpstreamLoop drives OSWorld's own run_single_example with the bridge, where a checkout of the upstream is at hand. No VM, no docker, no network, no model.""" from __future__ import annotations @@ -27,11 +30,14 @@ FIXTURES = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures") APP = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) PNG = base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC") +FOLDER = "/agent" +STUB = "import sys, types\nsys.modules['pyautogui'] = types.SimpleNamespace(FAILSAFE=True)\n" class FakeDesktop: - """A desktop of the test's, after the fake DesktopEnv of fixtures/osworld1_fake_desktop: its files in a folder of - the test's (relative paths only), python run there in a child process, one constant screen. Every operation is + """A desktop of the test's, after the fake DesktopEnv of fixtures/osworld1_fake_desktop: its files under a folder of + the test's (a desktop path /x is ROOT/x), python run there in a child process with a stand-in pyautogui and the + agent's folder's paths so placed, the folder made as the bridge asks, one constant screen. Every operation is recorded with its kind and start; two at once are counted as an overlap. ``fail`` names the operations that raise; ``silent`` makes python give no reply.""" @@ -58,15 +64,21 @@ def op(self, kind, **fields): self.busy -= 1 def at(self, path): - assert not os.path.isabs(path) and ".." not in path.split("/"), path - return os.path.join(self.root, path) + assert path.startswith("/") and ".." not in path.split("/"), path + return os.path.join(self.root, path.lstrip("/")) def python(self, code): - with self.op("python", code=code): + made = code == cp.MAKE_FOLDER.format(folder=FOLDER) + with self.op("python", code="folder" if made else code): if self.silent: return None - p = subprocess.run([sys.executable, "-I", "-c", code], cwd=self.root, capture_output=True, text=True, - timeout=30, stdin=subprocess.DEVNULL) + if made: + shutil.rmtree(self.at(FOLDER), True) + os.makedirs(self.at(FOLDER)) + return {"status": "success", "output": "made\n", "error": "", "returncode": 0} + code = code.replace(repr(FOLDER + "/")[:-1], repr(self.at(FOLDER) + "/")[:-1]) + p = subprocess.run([sys.executable, "-I", "-c", STUB + code], cwd=self.root, capture_output=True, + text=True, timeout=30, stdin=subprocess.DEVNULL) return {"status": "success" if p.returncode == 0 else "error", "output": p.stdout, "error": p.stderr, "returncode": p.returncode} @@ -188,16 +200,20 @@ def error(result): class Port(unittest.TestCase): """A unit: a fake desktop, the bridge over it, the server at a socket of the test's and the stand-in loop in a - thread, as a runner holds them.""" + thread, as a runner holds them. ``given``: the task is given at once (the window before it closed), as the run + does just after the agent's session opens.""" - def start(self, steps=3, pause=0.05, held=None, go=True): + def start(self, steps=3, pause=0.05, held=None, go=True, given=True): self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp, True) root = os.path.join(self.tmp, "desktop") os.mkdir(root) self.desktop = FakeDesktop(root) self.first = threading.Event() - self.bridge = cp.Bridge(self.desktop, steps, pause, (1920, 1080), 30, on_first=self.first.set, held=held) + self.bridge = cp.Bridge(self.desktop, steps, pause, (1920, 1080), FOLDER, on_first=self.first.set, held=held, + window=True) + if given: + self.bridge.close_window() folder = os.path.join(self.tmp, "socket") os.mkdir(folder, 0o700) self.socket, self.log = os.path.join(folder, "s"), os.path.join(self.tmp, "calls.jsonl") @@ -241,10 +257,10 @@ def test_initialize_and_the_tool_list_answer_before_the_desktop_is_ready_and_sta self.assertIn("1920 by 1080 pixels", init["instructions"]) self.assertIn("There are 3 steps in all", init["instructions"]) self.assertEqual([t["name"] for t in c.ask("tools/list")["result"]["tools"]], - ["info", "screenshot", "read", "python", "upload", "click", "move", "drag", "scroll", "type", - "key", "wait", "step", "done", "fail"]) + ["info", "screenshot", "upload", "prepare", "click", "move", "drag", "scroll", "type", "key", + "wait", "step", "done", "fail"]) self.assertEqual(value(c.call("info")), {"screen": [1920, 1080], "steps": 3, "steps_left": 3, "pause": 0.05, - "python_seconds": 30, "password": "password"}) + "password": "password", "folder": FOLDER}) shot = c.start("screenshot") with self.assertRaises(AssertionError): # held until the loop's first predict c.answer(shot, 0.5) @@ -254,12 +270,12 @@ def test_initialize_and_the_tool_list_answer_before_the_desktop_is_ready_and_sta self.assertTrue(self.first.is_set()) def test_the_reading_tools_take_no_step(self): - c = self.start(steps=3) + c = self.start(steps=3, given=False) self.assertEqual(image(c.call("screenshot")), PNG) - self.assertEqual(value(c.call("python", code="print(6 * 7)"))["output"], "42\n") - self.assertEqual(value(c.call("read", path="missing.txt")), {"data": None}) + self.assertEqual(value(c.call("prepare", actions=[], files=["missing.txt"])), {"files": {"missing.txt": None}}) self.assertEqual(value(c.call("upload", path="a.txt", data=base64.b64encode(b"abc").decode())), {"size": 3}) - self.assertEqual(value(c.call("read", path="a.txt")), {"data": base64.b64encode(b"abc").decode()}) + self.assertEqual(value(c.call("prepare", actions=[], files=["a.txt"])), + {"files": {"a.txt": base64.b64encode(b"abc").decode()}}) self.assertEqual(value(c.call("info"))["steps_left"], 3) self.assertEqual(self.ran, []) @@ -318,7 +334,7 @@ def test_the_loop_counts_one_step_for_each_acting_call_and_nothing_else(self): c.call("click", x=1, y=1) c.call("screenshot") c.call("step", actions=["pyautogui.press('a')", "pyautogui.press('b')"]) - c.call("python", code="pass") + c.call("info") last = c.call("key", keys=["enter"]) self.assertEqual(value(last), {"steps_left": 0, "done": True}) self.assertEqual(self.evaluated(), 3) @@ -330,8 +346,8 @@ def test_every_call_but_info_is_refused_past_the_budget_and_the_desktop_is_not_t c.call("click", x=1, y=1) self.evaluated() touched = len(self.desktop.events) - for tool, args in [("click", {"x": 1, "y": 1}), ("screenshot", {}), ("python", {"code": "pass"}), - ("read", {"path": "a"}), ("done", {})]: + for tool, args in [("click", {"x": 1, "y": 1}), ("screenshot", {}), ("upload", {"path": "a", "data": "YQ=="}), + ("prepare", {"actions": []}), ("done", {})]: self.assertEqual(error(c.call(tool, **args)), "the steps are used up", tool) self.assertEqual(value(c.call("info"))["steps_left"], 0) self.assertEqual(len(self.desktop.events), touched) @@ -357,18 +373,18 @@ def test_an_acting_call_is_answered_once_its_actions_and_their_pause_have_run(se answered = time.time() acted = next(e["start"] for e in self.desktop.events if e["kind"] == "action") self.assertGreaterEqual(answered - acted, 0.4) - self.assertEqual(self.desktop.kinds(), ["action", "capture"]) + self.assertEqual(self.desktop.kinds(), ["python", "action", "capture"]) # the agent's folder made first self.assertEqual(image(got), PNG) def test_calls_sent_during_a_step_wait_for_it_one_desktop_operation_at_a_time_in_their_order(self): c = self.start(steps=3, pause=0.3) - ids = [c.start("click", x=1, y=1), c.start("screenshot"), c.start("python", code="print(1)"), - c.start("read", path="none"), c.start("move", x=2, y=2)] + ids = [c.start("click", x=1, y=1), c.start("screenshot"), c.start("step", actions=[], files=["none"]), + c.start("move", x=2, y=2)] order = [] for rid in ids: c.answer(rid) order.append(rid) - self.assertEqual(self.desktop.kinds(), ["action", "capture", "capture", "python", "read", "action", "capture"]) + self.assertEqual(self.desktop.kinds(), ["python", "action", "capture", "read", "action", "capture"]) self.assertEqual(self.desktop.overlaps, 0) def test_the_answers_come_in_the_order_the_desktop_ran_the_calls(self): @@ -460,22 +476,18 @@ def test_a_turn_end_on_the_lifeline_ends_the_turn_and_its_close_stops_the_runner class Errors(Port): def test_a_failed_desktop_call_answers_with_its_words_and_the_next_succeeds(self): - c = self.start(steps=2) - self.desktop.fail = {"python", "capture"} - self.assertEqual(error(c.call("python", code="print(1)")), - "the desktop failed python: RuntimeError: python failed in the desktop") - self.assertIn("capture failed in the desktop", error(c.call("screenshot"))) + c = self.start(steps=2, given=False) + self.desktop.fail = {"upload"} + self.assertEqual(error(c.call("upload", path="a", data="YQ==")), + "the desktop failed upload: RuntimeError: upload failed in the desktop") + self.desktop.fail = set() + self.assertEqual(value(c.call("upload", path="a", data="YQ==")), {"size": 1}) + self.bridge.close_window() + self.desktop.fail = {"capture"} got = c.call("click", x=1, y=1) # the step ran; its screen could not be taken self.assertIn("the step was run; the screen could not be taken", error(got)) - self.desktop.fail = set() - self.assertEqual(value(c.call("python", code="print(1)"))["output"], "1\n") self.assertEqual(value(c.call("info"))["steps_left"], 1) - def test_python_with_no_reply_in_time_is_an_error(self): - c = self.start() - self.desktop.silent = True - self.assertEqual(error(c.call("python", code="pass")), "the desktop gave no reply within 30 s") - def test_an_unknown_tool_a_line_that_is_no_json_and_an_unknown_method_are_answered(self): c = self.start() self.assertEqual(c.ask("tools/call", {"name": "nope", "arguments": {}})["error"]["code"], -32602) @@ -490,10 +502,10 @@ def test_a_loop_that_fails_during_a_step_has_the_call_answered(self): self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp, True) desktop = FakeDesktop(self.tmp) - bridge = cp.Bridge(desktop, 3, 0, (100, 100), 30) + bridge = cp.Bridge(desktop, 3, 0, (100, 100), FOLDER) answers = [] call = cp.Call("click", {"x": 1, "y": 1}, lambda c, v, i, e: answers.append(e)) - cp.checked(call, bridge.screen) + cp.checked(call, bridge.screen, bridge.folder) bridge.submit(call) _, actions = bridge.predict("x", {}) next(iter(actions)) # the loop takes the action, then fails in it @@ -503,17 +515,18 @@ def test_a_loop_that_fails_during_a_step_has_the_call_answered(self): class Upload(Port): def test_an_upload_in_chunks_is_the_file_with_its_size_so_far(self): - c = self.start() + c = self.start(given=False) parts = [b"first", b"-second", b"-third"] sizes = [value(c.call("upload", path="f.bin", data=base64.b64encode(p).decode(), append=i > 0))["size"] for i, p in enumerate(parts)] self.assertEqual(sizes, [5, 12, 18]) - self.assertEqual(value(c.call("read", path="f.bin")), {"data": base64.b64encode(b"".join(parts)).decode()}) - self.assertEqual(sorted(os.listdir(self.desktop.root)), ["f.bin"]) + self.assertEqual(value(c.call("prepare", actions=[], files=["f.bin"])), + {"files": {"f.bin": base64.b64encode(b"".join(parts)).decode()}}) + self.assertEqual(sorted(os.listdir(self.desktop.at(FOLDER))), ["f.bin"]) self.assertEqual(self.ran, []) def test_a_chunk_over_four_mib_is_refused(self): - c = self.start() + c = self.start(given=False) big = base64.b64encode(b"x" * (cp.CHUNK_BYTES + 1)).decode() self.assertEqual(error(c.call("upload", path="f.bin", data=big)), f"data is at most {cp.CHUNK_BYTES} bytes a call") self.assertNotIn("upload", self.desktop.kinds()) @@ -523,18 +536,18 @@ def test_a_file_sent_once_for_a_run_reaches_the_next_units_desktop_by_its_sha256 self.addCleanup(shutil.rmtree, run, True) content = os.urandom(3000) digest = hashlib.sha256(content).hexdigest() - c = self.start(held=cp.Held(run)) # the first unit sends the bytes + c = self.start(held=cp.Held(run), given=False) # the first unit sends the bytes self.assertIn("the run holds no file of that sha256", error(c.call("upload", path="tool.bin", sha256=digest))) c.call("upload", path="tool.bin", data=base64.b64encode(content[:1000]).decode(), sha256=digest) c.call("upload", path="tool.bin", data=base64.b64encode(content[1000:]).decode(), append=True) self.assertEqual(os.listdir(run), [digest]) second = Port("run") # the next unit of the same run sends none second.addCleanup = self.addCleanup - c2 = second.start(held=cp.Held(run)) + c2 = second.start(held=cp.Held(run), given=False) self.assertEqual(value(c2.call("upload", path="tool.bin", sha256=digest)), {"size": 3000}) - with open(os.path.join(second.desktop.root, "tool.bin"), "rb") as f: + with open(second.desktop.at(FOLDER + "/tool.bin"), "rb") as f: self.assertEqual(f.read(), content) - self.assertEqual(second.desktop.kinds(), ["upload"]) + self.assertEqual(second.desktop.kinds(), ["python", "upload"]) # the folder made, the upload with open(os.path.join(run, digest), "r+b") as f: # a kept file that changed is not taken f.write(b"\0") self.assertIn("the run holds no file of that sha256", error(c2.call("upload", path="t2.bin", sha256=digest))) @@ -542,12 +555,12 @@ def test_a_file_sent_once_for_a_run_reaches_the_next_units_desktop_by_its_sha256 def test_chunks_that_do_not_hash_to_the_sha256_given_are_not_kept(self): run = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, run, True) - c = self.start(held=cp.Held(run)) + c = self.start(held=cp.Held(run), given=False) c.call("upload", path="t.bin", data=base64.b64encode(b"abc").decode(), sha256="0" * 64) self.assertEqual(os.listdir(run), []) def test_with_no_folder_for_the_run_an_upload_by_sha256_alone_is_refused(self): - c = self.start() + c = self.start(given=False) self.assertEqual(error(c.call("upload", path="t.bin", sha256="0" * 64)), "the run holds no file of that sha256: send its data") @@ -575,10 +588,10 @@ def test_a_connection_whose_input_ends_is_closed_once_its_calls_are_answered(sel class CallLog(Port): def test_every_tool_call_is_recorded_with_its_tool_times_and_failure_and_nothing_else(self): - c = self.start(steps=2, pause=0.3) + c = self.start(steps=3, pause=0.3) c.call("info") c.call("screenshot") - c.call("python", code="print('a-secret-text')") + c.call("step", actions=["print('a-secret-text')"]) c.call("click", x=1, y=1) c.call("click", x=99999, y=1) c.ask("tools/call", {"name": "nope"}) @@ -586,7 +599,7 @@ def test_every_tool_call_is_recorded_with_its_tool_times_and_failure_and_nothing c.call("screenshot") lines = self.logged() self.assertEqual([(x["tool"], x["isError"]) for x in lines], - [("info", False), ("screenshot", False), ("python", False), ("click", False), ("click", True), + [("info", False), ("screenshot", False), ("step", False), ("click", False), ("click", True), (None, True), ("done", False), ("screenshot", True)]) for x in lines: self.assertEqual(set(x), {"tool", "start", "end", "isError"}) @@ -735,7 +748,8 @@ def bridge(self, steps=5, held=None): tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmp, True) self.desktop = FakeDesktop(tmp) - b = cp.Bridge(self.desktop, steps, 0.01, (100, 100), 30, held=held) + os.makedirs(self.desktop.at(FOLDER)) + b = cp.Bridge(self.desktop, steps, 0.01, (100, 100), FOLDER, held=held) self.addCleanup(b.finish) return b @@ -750,7 +764,7 @@ def call(self, b, tool, **args): answers = [] call = cp.Call(tool, args, lambda c, v, i, e: answers.append((v, i, e))) call.answers = answers - cp.checked(call, b.screen) + cp.checked(call, b.screen, b.folder) b.submit(call) return call @@ -761,7 +775,7 @@ def wire(self, conn=None): def upload(self, b, **args): call = cp.Call("upload", args, None) - cp.checked(call, b.screen) + cp.checked(call, b.screen, b.folder) return b._upload(call) @@ -773,11 +787,10 @@ def test_acting_answers_and_screenshot_coordinates_are_described(self): description = listed[name]["description"] if name in cp.FINAL: self.assertIn("Returns {steps_left, done}, with no image", description) + elif name == "step": + self.assertIn("Returns the step's observation as a PNG image when screenshot is true", description) else: - self.assertIn("Returns the screen as a PNG image after the", description) - self.assertIn("pause", description) - if name == "step": - self.assertIn("when screenshot is true", description) + self.assertIn("Returns the screen as a PNG image after the pause", description) for name in ("click", "move", "drag", "scroll"): props = listed[name]["inputSchema"]["properties"] for point, edge in (("x", "left"), ("y", "top"), ("to_x", "left"), ("to_y", "top")): @@ -790,7 +803,8 @@ def test_acting_answers_and_screenshot_coordinates_are_described(self): self.assertIn("x and y in screenshot pixels at the screen's size, from its top left corner", instructions) self.assertIn("Look at a screenshot before the first pointing action", instructions) self.assertIn("After the actions and their 0.01 s pauses", instructions) - self.assertIn("click, move, drag, scroll, type, key and wait return the screen as a PNG image", instructions) + self.assertIn("click, move, drag, scroll, type, key and wait return the step's observation, the screen as a PNG " + "image", instructions) self.assertIn("step does when screenshot is true", instructions) self.assertIn("done and fail answer before they run, with no image", instructions) self.assertIn("A step with end, done or fail, runs its actions and then ends the task, in that one step", @@ -912,8 +926,8 @@ def test_json_too_deep_to_read_is_answered_and_the_connection_closed(self): class Envelope(Bare): - VALID = {"info": {}, "screenshot": {}, "read": {"path": "a"}, "python": {"code": "pass"}, - "upload": {"path": "a", "data": "YQ=="}, "click": {"x": 1, "y": 2}, "move": {"x": 1, "y": 2}, + VALID = {"info": {}, "screenshot": {}, "upload": {"path": "a", "data": "YQ=="}, "prepare": {"actions": ["WAIT"]}, + "click": {"x": 1, "y": 2}, "move": {"x": 1, "y": 2}, "drag": {"x": 1, "y": 2, "to_x": 3, "to_y": 4}, "scroll": {"x": 1, "y": 2, "dy": 1}, "type": {"text": "a"}, "key": {"keys": ["a"]}, "wait": {}, "step": {"actions": ["WAIT"]}, "done": {}, "fail": {}} @@ -942,16 +956,16 @@ def test_an_argument_the_schema_does_not_list_is_refused(self): s.call(1, {"name": "wait", "arguments": {"extra": 1}}, self.wire()) s.call(2, {"name": "click", "arguments": {"x": 1, "y": 1, "clicks": 2}}, self.wire()) self.assertEqual(([r["isError"] for r in s.records], b.booked), ([True, True], 0)) - self.assertEqual([t["inputSchema"]["additionalProperties"] for t in cp.listed((100, 100))], [False] * 15) + self.assertEqual([t["inputSchema"]["additionalProperties"] for t in cp.listed((100, 100))], [False] * 14) def test_the_patterns_mean_the_same_in_python_and_in_the_listed_schema(self): for actions in (["\ufeff"], ["\u0085"], ["\u00a0"], ["a"]): # not blank, read alike either way - cp.checked(cp.Call("step", {"actions": actions}, None), (100, 100)) + cp.checked(cp.Call("step", {"actions": actions}, None), (100, 100), FOLDER) for actions in ([" \t\r\n\f\v"], ["DONE\n"], [" FAIL "]): with self.subTest(actions=actions), self.assertRaises(cp.Refused): - cp.checked(cp.Call("step", {"actions": actions}, None), (100, 100)) + cp.checked(cp.Call("step", {"actions": actions}, None), (100, 100), FOLDER) with self.assertRaisesRegex(cp.Refused, "64 lowercase"): # $ is the text's end - cp.checked(cp.Call("upload", {"path": "f", "sha256": "0" * 64 + "\n"}, None), (100, 100)) + cp.checked(cp.Call("upload", {"path": "f", "sha256": "0" * 64 + "\n"}, None), (100, 100), FOLDER) patterns = json.dumps(cp.listed((100, 100))) self.assertNotRegex(patterns, r"\\\\[sSdDwWbB]", "a class whose meaning differs between engines") @@ -981,22 +995,26 @@ def test_each_listed_schema_is_what_the_calls_are_checked_against(self): self.assertEqual(sorted(listed), sorted(self.VALID)) wrong = {"integer": "1", "string": 1, "boolean": "true", "array": "a"} for tool, schema in listed.items(): - cp.checked(cp.Call(tool, dict(self.VALID[tool]), None), (100, 100)) # the valid call holds + cp.checked(cp.Call(tool, dict(self.VALID[tool]), None), (100, 100), FOLDER) # the valid call holds for name in schema.get("required", []): with self.subTest(tool=tool, missing=name), self.assertRaises(cp.Refused): - cp.checked(cp.Call(tool, {k: v for k, v in self.VALID[tool].items() if k != name}, None), (100, 100)) + cp.checked(cp.Call(tool, {k: v for k, v in self.VALID[tool].items() if k != name}, None), (100, 100), + FOLDER) for name, prop in schema["properties"].items(): with self.subTest(tool=tool, wrong=name), self.assertRaises(cp.Refused): - cp.checked(cp.Call(tool, dict(self.VALID[tool], **{name: wrong[prop["type"]]}), None), (100, 100)) + cp.checked(cp.Call(tool, dict(self.VALID[tool], **{name: wrong[prop["type"]]}), None), (100, 100), + FOLDER) if "maximum" in prop: with self.subTest(tool=tool, over=name), self.assertRaises(cp.Refused): - cp.checked(cp.Call(tool, dict(self.VALID[tool], **{name: prop["maximum"] + 1}), None), (100, 100)) - cp.checked(cp.Call(tool, dict(self.VALID[tool], **{name: prop["maximum"]}), None), (100, 100)) + cp.checked(cp.Call(tool, dict(self.VALID[tool], **{name: prop["maximum"] + 1}), None), + (100, 100), FOLDER) + cp.checked(cp.Call(tool, dict(self.VALID[tool], **{name: prop["maximum"]}), None), (100, 100), FOLDER) def test_an_answer_over_the_bound_is_recorded_as_failed(self): b = self.bridge() + b.seen, b.unseen = b"x" * 2000, "" # an observation too large for the bound below s, conn = self.server(b), FakeConnection() - s.call(1, {"name": "python", "arguments": {"code": "print('x' * 2000)"}}, self.wire(conn)) + s.call(1, {"name": "screenshot", "arguments": {}}, self.wire(conn)) with mock.patch.object(cp, "LINE_BYTES", 512): b._serve(b.calls.popleft()) self.assertTrue(until(lambda: conn.sent)) @@ -1078,9 +1096,9 @@ def test_an_append_leaves_a_file_named_like_its_scratch_alone(self): self.upload(b, path="file", data=b64(b"A")) self.upload(b, path="file.part", data=b64(b"unrelated")) self.assertEqual(self.upload(b, path="file", data=b64(b"B"), append=True), {"size": 2}) - with open(os.path.join(self.desktop.root, "file.part"), "rb") as f: + with open(self.desktop.at(FOLDER + "/file.part"), "rb") as f: self.assertEqual(f.read(), b"unrelated") - self.assertEqual(sorted(os.listdir(self.desktop.root)), ["file", "file.part"]) + self.assertEqual(sorted(os.listdir(self.desktop.at(FOLDER))), ["file", "file.part"]) def held(self): run = tempfile.mkdtemp() @@ -1096,7 +1114,7 @@ def test_a_file_taken_by_its_sha256_starts_the_file_and_append_is_refused(self): with self.assertRaisesRegex(cp.Refused, "append needs data"): self.upload(b, path="f", sha256=digest, append=True) self.assertEqual(self.upload(b, path="f", sha256=digest), {"size": 1}) - with open(os.path.join(self.desktop.root, "f"), "rb") as f: + with open(self.desktop.at(FOLDER + "/f"), "rb") as f: self.assertEqual(f.read(), b"B") def test_a_file_taken_by_its_sha256_drops_what_was_being_kept_for_its_path(self): @@ -1107,7 +1125,7 @@ def test_a_file_taken_by_its_sha256_drops_what_was_being_kept_for_its_path(self) self.upload(b, path="f", data=base64.b64encode(b"A").decode(), sha256=hash_ac) self.upload(b, path="f", sha256=hash_b) self.upload(b, path="f", data=base64.b64encode(b"C").decode(), append=True) - with open(os.path.join(self.desktop.root, "f"), "rb") as f: + with open(self.desktop.at(FOLDER + "/f"), "rb") as f: self.assertEqual(f.read(), b"BC") self.assertIsNone(held.get(hash_ac)) @@ -1128,7 +1146,7 @@ def test_the_listed_bound_of_data_is_the_checks(self): args = {"path": "f", "data": base64.b64encode(b"a" * size).decode()} listed = cp._passes(args, schema, "upload", "arguments") try: - cp.checked(cp.Call("upload", args, None), (100, 100)) + cp.checked(cp.Call("upload", args, None), (100, 100), FOLDER) taken = True except cp.Refused: taken = False @@ -1151,7 +1169,7 @@ def python(code): self.desktop.python = python with self.assertRaises(TimeoutError): self.upload(b, path="f", data=b64(b"B"), append=True) - self.assertEqual(sorted(os.listdir(self.desktop.root)), ["f"]) + self.assertEqual(sorted(os.listdir(self.desktop.at(FOLDER))), ["f"]) def test_a_stop_as_a_file_is_kept_leaves_no_file_and_no_descriptor(self): held = self.held() @@ -1241,7 +1259,7 @@ def password(self): return env.client_password -bridge = cp.Bridge(Desktop(), int(steps), 0.1, (1920, 1080), 90, on_first=lambda: cp.ready(ready, ok=True)) +bridge = cp.Bridge(Desktop(), int(steps), 0.1, (1920, 1080), "/tmp/agent", on_first=lambda: cp.ready(ready, ok=True)) server = cp.Server(bridge, sock, log) threading.Thread(target=cp.lifeline, args=(int(lifeline), bridge), daemon=True).start() example = {"id": "task-1", "instruction": "Open the terminal.", "config": [], "evaluator": {"func": "check"}} @@ -1357,5 +1375,457 @@ def test_a_turn_that_ends_with_no_done_has_the_desktop_evaluated_as_it_is(self): ["pyautogui.click(10, 20, clicks=1, button='left')"]) +def shot(n): + return PNG + b"#%d" % n + + +class ChangingDesktop(FakeDesktop): + """The fake desktop with a screen that changes with each action run.""" + + def __init__(self, root): + super().__init__(root) + self.screen_no = 0 + + def act(self, action): + super().act(action) + self.screen_no += 1 # the screen moves on, whatever the action + + def capture(self): + with self.op("capture"): + return shot(self.screen_no) + + def captures(self): + return self.kinds().count("capture") + + +class Kept(ChangingDesktop): + """A desktop whose benchmark keeps its own observation, as OSWorld's loop does: the bridge never captures.""" + last = None + + def observation(self): + return self.last + + +def observing_loop(agent, desktop, steps, pause, ran, go, hands): + """OSWorld's loop in a few lines: the start picture, then one predict a step, its actions run with the pause after + each and, when the loop ``hands`` its observation, a picture after each action that the next predict gets.""" + agent.reset(None, vm_ip="127.0.0.1") + go.wait(30) + obs = {"screenshot": desktop.capture()} if hands else None + done, step = False, 0 + while not done and step < steps: + _, actions = agent.predict("the task", obs) + for action in actions: + desktop.act(action) + time.sleep(pause) + ran.append(action) + if hands: + obs = {"screenshot": desktop.capture()} + desktop.last = obs["screenshot"] + done = action in ("DONE", "FAIL") + if done: + break + step += 1 + ran.append(("evaluated", step)) + + +class Changing(unittest.TestCase): + def start(self, steps=5, hands=True, window=True, given=True): + """A unit whose screen changes with each action: the bridge over the desktop, its server and the loop. + ``given``: the task is given at once (the window closed), as the run does just after the agent's session + opens.""" + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + root = os.path.join(self.tmp, "desktop") + os.mkdir(root) + self.desktop = (Kept if hands else ChangingDesktop)(root) + self.prepared = os.path.join(self.tmp, "prepared.jsonl") + self.bridge = cp.Bridge(self.desktop, steps, 0.01, (1920, 1080), FOLDER, window=window, prepared=self.prepared) + folder = os.path.join(self.tmp, "socket") + os.mkdir(folder, 0o700) + self.socket, self.log = os.path.join(folder, "s"), os.path.join(self.tmp, "calls.jsonl") + self.server = cp.Server(self.bridge, self.socket, self.log) + self.addCleanup(self.server.close) + self.ran, self.go = [], threading.Event() + self.go.set() + self.loop = threading.Thread(target=self._loop, args=(steps, hands), daemon=True) + self.loop.start() + self.addCleanup(self.bridge.finish) + c = Client(self.socket) + self.addCleanup(c.close) + c.initialize() + c.call("screenshot") # served once the loop's first predict has begun + if given: + self.bridge.close_window() + return c + + def _loop(self, steps, hands): + try: + observing_loop(self.bridge, self.desktop, steps, 0.01, self.ran, self.go, hands) + finally: + self.bridge.finish() + + def evaluated(self, seconds=10): + self.loop.join(seconds) + self.assertFalse(self.loop.is_alive(), "the loop did not end") + return self.ran[-1][1] + + def left(self, c): + return value(c.call("info"))["steps_left"] + + def put(self, path, data): + with open(self.desktop.at(path), "wb") as f: + f.write(data) + + +class ToolList(Changing): + def test_the_tool_list_has_no_python_and_no_read_and_names_prepare_and_a_steps_files(self): + c = self.start(steps=3) + listed = {t["name"]: t for t in c.ask("tools/list")["result"]["tools"]} + self.assertEqual(list(listed), ["info", "screenshot", "upload", "prepare", "click", "move", "drag", "scroll", + "type", "key", "wait", "step", "done", "fail"]) + self.assertNotIn("maxItems", listed["step"]["inputSchema"]["properties"]["files"]) + self.assertNotIn("maxItems", listed["prepare"]["inputSchema"]["properties"]["files"]) + self.assertEqual(sorted(listed["prepare"]["inputSchema"]["properties"]), ["actions", "files"]) + init = c.initialize()["result"] + self.assertEqual(init["serverInfo"], {"name": "computer", "version": "1"}) + self.assertIn("upload and prepare work inside folder (/agent)", init["instructions"]) + self.assertEqual(value(c.call("info")), {"screen": [1920, 1080], "steps": 3, "steps_left": 3, "pause": 0.01, + "password": "password", "folder": FOLDER}) + for gone in ("python", "read"): + answer = c.ask("tools/call", {"name": gone, "arguments": {}}) + self.assertEqual(answer["error"]["message"], f"no tool {gone!r}") + + def test_a_bridge_needs_a_plain_absolute_folder(self): + for folder in (None, "", "agent", "/", "/agent/", "/a/../b"): + with self.subTest(folder=folder), self.assertRaises(ValueError): + cp.Bridge(FakeDesktop(self.id()), 1, 0, (10, 10), folder) + + +class Observation(Changing): + def test_before_the_first_step_screenshot_is_the_loops_start_picture_and_takes_no_new_one(self): + c = self.start(steps=3) + first, again = image(c.call("screenshot")), image(c.call("screenshot")) + self.assertEqual((first, again), (shot(0), shot(0))) + self.assertEqual(self.desktop.captures(), 1) # the loop's own: the bridge took none + + def test_two_screenshots_with_no_step_between_are_the_same_and_a_step_that_acts_changes_them(self): + for hands in (True, False): + with self.subTest(hands=hands): + c = self.start(steps=5, hands=hands) + before = image(c.call("screenshot")) + got = c.call("click", x=1, y=1) + after = image(c.call("screenshot")) + self.assertEqual((image(got), after, image(c.call("screenshot"))), (after, after, after)) + self.assertNotEqual(before, after) + got = c.call("step", actions=["pyautogui.press('a')", "pyautogui.press('b')"], screenshot=True) + self.assertEqual(image(got), image(c.call("screenshot"))) + self.assertNotEqual(image(got), after) + self.assertIsNone(image(c.call("step", actions=["pyautogui.press('c')"]))) + + def test_an_empty_step_is_one_step_and_leaves_the_observation_as_it_was(self): + for hands in (True, False): + with self.subTest(hands=hands): + c = self.start(steps=5, hands=hands) + c.call("click", x=1, y=1) + seen, captures = image(c.call("screenshot")), self.desktop.captures() + got = c.call("step", actions=[], screenshot=True) + self.assertEqual((value(got), image(got)), ({"steps_left": 3, "done": False}, seen)) + self.assertEqual(self.desktop.captures(), captures) + + def test_wait_is_one_step_and_a_fresh_observation(self): + for hands in (True, False): + with self.subTest(hands=hands): + c = self.start(steps=5, hands=hands) + before = image(c.call("screenshot")) + got = c.call("wait") + self.assertEqual(value(got), {"steps_left": 4, "done": False}) + self.assertNotEqual(image(got), before) + self.assertEqual(image(c.call("screenshot")), image(got)) + self.assertEqual(self.ran, ["WAIT"]) + + def test_with_its_own_observation_the_bridge_takes_no_picture_and_without_one_it_takes_one_a_step(self): + c = self.start(steps=5, hands=True) + for _ in range(3): + c.call("click", x=1, y=1) + c.call("screenshot") + self.assertEqual(self.desktop.captures(), 4) # the loop's start picture and one per action + c = self.start(steps=5, hands=False) + for _ in range(3): + c.call("step", actions=["pyautogui.press('a')", "pyautogui.press('b')"]) + c.call("screenshot") + self.assertEqual(self.desktop.captures(), 4) # the start picture and one as each step ended + self.assertEqual(self.desktop.kinds()[-1], "capture") + + def test_a_step_whose_picture_could_not_be_taken_says_so_and_screenshot_too(self): + c = self.start(steps=5, hands=False) + self.desktop.fail = {"capture"} + self.assertIn("the step was run; the screen could not be taken", error(c.call("click", x=1, y=1))) + self.assertIn("there is no observation", error(c.call("screenshot"))) + self.desktop.fail = set() + self.assertEqual(image(c.call("click", x=1, y=1)), image(c.call("screenshot"))) + + +class Window(Changing): + def test_upload_and_prepare_work_in_the_window_and_take_no_step(self): + c = self.start(steps=3, given=False) + self.assertEqual(value(c.call("upload", path="/agent/run.py", data=base64.b64encode(b"print(1)").decode())), + {"size": 8}) + self.assertEqual(value(c.call("upload", path="x", data=base64.b64encode(b"abc").decode())), {"size": 3}) + with open(self.desktop.at("/agent/x"), "rb") as f: + self.assertEqual(f.read(), b"abc") # a relative path is the folder's + got = c.call("prepare", actions=["open('agent/out.txt', 'w').write(open('agent/run.py').read() * 2)", "WAIT"], + files=["/agent/out.txt", "/agent/none"]) + self.assertEqual(value(got), {"files": {"/agent/out.txt": base64.b64encode(b"print(1)print(1)").decode(), + "/agent/none": None}}) + self.assertIsNone(image(got)) + self.assertEqual(value(c.call("prepare", actions=[])), {"files": {}}) + self.assertEqual(self.left(c), 3) + self.assertEqual(self.ran, []) + with open(self.prepared) as f: + said = [json.loads(line) for line in f] + self.assertEqual([(s["actions"], s["files"]) for s in said], + [(["open('agent/out.txt', 'w').write(open('agent/run.py').read() * 2)", "WAIT"], + ["/agent/out.txt", "/agent/none"]), ([], [])]) + + def test_an_acting_call_in_the_window_is_refused_and_takes_no_step(self): + c = self.start(steps=3, given=False) + for tool, args in (("click", {"x": 1, "y": 1}), ("step", {"actions": []}), ("wait", {}), ("done", {})): + with self.subTest(tool=tool): + self.assertIn("the task has not been given yet", error(c.call(tool, **args))) + self.assertEqual(self.left(c), 3) + self.bridge.close_window() + self.assertEqual(value(c.call("click", x=1, y=1)), {"steps_left": 2, "done": False}) + self.assertEqual(self.ran, ["pyautogui.click(1, 1, clicks=1, button='left')"]) + + def test_once_the_task_is_given_upload_and_prepare_are_refused(self): + c = self.start(steps=3, given=False) + c.call("prepare", actions=[]) + self.bridge.close_window() + self.assertEqual(error(c.call("upload", path="/agent/a", data="YQ==")), + "the task has been given: upload is taken only before it") + self.assertEqual(error(c.call("prepare", actions=["print(1)"])), + "the task has been given: prepare is taken only before it") + self.assertEqual(self.left(c), 3) + self.assertEqual(len(self.desktop.kinds()), 2) # the folder made, the start picture + + def test_the_lifelines_task_given_closes_the_window(self): + c = self.start(steps=3, given=False) + read, write = os.pipe() + with mock.patch.object(cp.os, "kill"): + reader = threading.Thread(target=cp.lifeline, args=(read, self.bridge), daemon=True) + reader.start() + os.write(write, cp.TASK_GIVEN + b"\n") + deadline = time.monotonic() + 5 + while self.bridge.window and time.monotonic() < deadline: + time.sleep(0.01) + self.assertIn("the task has been given", error(c.call("prepare", actions=[]))) + os.close(write) + reader.join(5) + os.close(read) + + def test_with_no_window_upload_and_prepare_are_refused_from_the_start(self): + c = self.start(steps=3, window=False, given=False) + self.assertIn("the task has been given", error(c.call("upload", path="/agent/a", data="YQ=="))) + self.assertEqual(value(c.call("click", x=1, y=1)), {"steps_left": 2, "done": False}) + + def test_paths_outside_the_folder_are_refused_at_once_and_take_no_step(self): + c = self.start(steps=3, given=False) + for path in ("/etc/passwd", "/agent", "/agent/", "/agent/../x", "../x", "./x", "/agentx/y", "/agent//x"): + with self.subTest(path=path): + self.assertIn("is a path inside /agent", error(c.call("upload", path=path, data="YQ=="))) + self.assertIn("files[1] is a path inside /agent", + error(c.call("prepare", actions=[], files=["/agent/a", path]))) + self.bridge.close_window() + self.assertIn("files[0] is a path inside /agent", error(c.call("step", actions=["x()"], files=["/tmp/a"]))) + self.assertEqual(self.left(c), 3) + self.assertEqual((self.ran, len(self.desktop.kinds())), ([], 2)) + + def test_a_prepare_action_the_desktop_did_not_end_in_time_stops_it(self): + c = self.start(steps=3, given=False) + self.desktop.silent = True + self.assertEqual(error(c.call("prepare", actions=["print(1)", "print(2)"])), + "the desktop did not end actions[0] in time; the actions after it were not run") + self.assertEqual(self.left(c), 3) + + +class WindowClosure(unittest.TestCase): + def setUp(self): + tmp = self.enterContext(tempfile.TemporaryDirectory()) + self.desktop = ChangingDesktop(tmp) + self.bridge = cp.Bridge(self.desktop, 3, 0, (100, 100), FOLDER, window=True) + self.bridge._begin(None) + self.addCleanup(self.bridge.finish) + + def submit(self, tool): + args = {"actions": ["print(1)"], "files": ["/agent/a"]} if tool == "prepare" else { + "path": "/agent/a", "data": "YQ=="} + replies = [] + call = cp.Call(tool, args, lambda c, v, i, e: replies.append((v, i, e))) + cp.checked(call, self.bridge.screen, self.bridge.folder) + self.bridge.submit(call) + return call, replies + + def test_closing_refuses_queued_upload_and_prepare_as_calls_sent_after_it(self): + pending = [self.submit(tool) for tool in cp.WINDOW] + before = list(self.desktop.events) + self.bridge.close_window() + for call, replies in pending: + _, later = self.submit(call.tool) + self.assertEqual(replies, later) + self.assertEqual(replies, [(None, None, f"the task has been given: {call.tool} is taken only before it")]) + self.assertEqual(list(self.bridge.calls), []) + self.assertEqual(self.desktop.events, before) + self.assertEqual(self.bridge.used, 0) + + def test_a_dequeued_call_is_refused_if_it_has_not_started_before_closure(self): + for tool in cp.WINDOW: + with self.subTest(tool=tool): + self.bridge.window = True + call, replies = self.submit(tool) + self.assertIs(self.bridge._next(), call) + before = list(self.desktop.events) + self.bridge.close_window() + self.bridge._serve(call) + _, later = self.submit(tool) + self.assertEqual(replies, later) + self.assertEqual(self.desktop.events, before) + + def test_closing_waits_for_running_upload_and_prepare_even_when_they_fail(self): + for tool in cp.WINDOW: + for fails in (False, True): + with self.subTest(tool=tool, fails=fails): + self.bridge.window = True + call, replies = self.submit(tool) + entered, release, waiting, closed = (threading.Event() for _ in range(4)) + operation = "upload" if tool == "upload" else ("python" if fails else "read") + original = getattr(self.desktop, operation) + wait = self.bridge.change.wait + + def blocked(*args): + entered.set() + if not release.wait(5): + raise RuntimeError("the test did not release the desktop") + if fails: + raise RuntimeError("the desktop operation failed") + return original(*args) + + def waited(*args): + waiting.set() + return wait(*args) + + def close(): + self.bridge.close_window() + closed.set() + + with mock.patch.object(self.desktop, operation, blocked), \ + mock.patch.object(self.bridge.change, "wait", waited): + serving = threading.Thread(target=lambda: self.bridge._serve(self.bridge._next())) + closing = threading.Thread(target=close) + serving.start() + try: + self.assertTrue(entered.wait(5)) + closing.start() + self.assertTrue(waiting.wait(1), "closure did not wait for the running call") + self.assertFalse(closed.is_set()) + self.assertFalse(self.bridge.window) + _, later = self.submit(tool) + self.assertIn("the task has been given", later[0][2]) + finally: + release.set() + serving.join(5) + if closing.ident is not None: + closing.join(5) + self.assertFalse(serving.is_alive() or closing.is_alive()) + self.assertTrue(closed.is_set()) + self.assertEqual(len(replies), 1) + self.assertEqual(replies[0][2] is not None, fails) + self.assertEqual(self.bridge.used, 0) + + +class StepFiles(Changing): + def test_a_step_returns_each_file_null_for_a_missing_one_and_an_error_for_one_too_large(self): + c = self.start(steps=3) + self.put("/agent/small", b"hello") + self.put("/agent/big", b"x" * (cp.READ_BYTES + 1)) + got = c.call("step", actions=["pyautogui.press('a')"], files=["/agent/small", "/agent/missing", "big"]) + self.assertEqual(value(got), {"steps_left": 2, "done": False, "files": { + "/agent/small": base64.b64encode(b"hello").decode(), "/agent/missing": None, + "big": {"error": f"over {cp.READ_BYTES} bytes"}}}) + self.desktop.fail = {"read"} + got = c.call("step", actions=[], files=["/agent/small"]) + self.assertEqual(value(got), {"steps_left": 1, "done": False, "files": {"/agent/small": { + "error": "the desktop could not read it: RuntimeError: read failed in the desktop"}}}) + + def test_forty_small_files_come_back_in_one_step(self): + c = self.start(steps=3) + names = [f"/agent/f{i}" for i in range(40)] + for i, name in enumerate(names): + self.put(name, b"file %d" % i) + got = value(c.call("step", actions=["pyautogui.press('a')"], files=names)) + self.assertEqual(got, {"steps_left": 2, "done": False, + "files": {name: base64.b64encode(b"file %d" % i).decode() + for i, name in enumerate(names)}}) + self.assertEqual(self.left(c), 2) + + def test_files_that_would_take_the_answer_past_its_bound_are_errors_and_the_others_come(self): + c = self.start(steps=3) + big = b"y" * (11 * 1024 * 1024) # 11 MiB: two of them pass 24 MiB in base64 + for name in ("a", "b", "c"): + self.put(f"/agent/{name}", big) + self.put("/agent/small", b"hi") + got = c.call("step", seconds=60, actions=["pyautogui.press('a')"], screenshot=True, + files=["/agent/a", "/agent/b", "/agent/small", "/agent/c"]) + self.assertFalse(got["isError"]) + over = {"error": f"the answer would pass {cp.LINE_BYTES} bytes with this file"} + self.assertEqual(value(got)["files"], {"/agent/a": base64.b64encode(big).decode(), "/agent/b": over, + "/agent/small": base64.b64encode(b"hi").decode(), "/agent/c": over}) + self.assertEqual(image(got), shot(1)) + self.assertEqual(self.left(c), 2) + + def test_the_files_are_read_once_the_actions_and_their_pauses_have_run(self): + c = self.start(steps=3) + c.call("step", actions=["pyautogui.press('a')", "WAIT"], files=["/agent/a"]) + kinds = self.desktop.kinds() + self.assertEqual(kinds[kinds.index("action"):], ["action", "capture", "action", "capture", "read"]) + + def test_every_call_that_reaches_the_desktop_lowers_steps_left_but_upload_and_prepare_in_the_window(self): + c = self.start(steps=9, given=False) + c.call("upload", path="/agent/a", data="YQ==") + c.call("prepare", actions=["print(1)"]) + c.call("screenshot") + self.assertEqual(self.left(c), 9) + self.bridge.close_window() + calls = [("click", {"x": 1, "y": 1}), ("move", {"x": 1, "y": 1}), ("drag", {"x": 1, "y": 1, "to_x": 2, "to_y": 2}), + ("scroll", {"x": 1, "y": 1, "dy": 1}), ("type", {"text": "a"}), ("key", {"keys": ["a"]}), + ("wait", {}), ("step", {"actions": [], "files": ["/agent/a"]})] + for i, (tool, args) in enumerate(calls, 1): + self.assertEqual(value(c.call(tool, **args))["steps_left"], 9 - i, tool) + c.call("screenshot") + self.assertEqual(self.left(c), 9 - i, tool) + self.assertEqual(value(c.call("done")), {"steps_left": 0, "done": True}) + self.assertEqual(self.evaluated(), 9) + + def test_a_step_with_an_end_and_files_runs_its_actions_and_ends_the_task(self): + c = self.start(steps=5) + got = c.call("step", actions=["pyautogui.press('a')"], end="done", files=["/agent/a"], screenshot=True) + self.assertEqual(value(got), {"steps_left": 4, "done": True, "files": {"/agent/a": None}}) + self.assertEqual(image(got), shot(1)) + self.assertEqual(self.evaluated(), 1) + self.assertEqual(self.ran, ["pyautogui.press('a')", "DONE", ("evaluated", 1)]) + self.assertEqual(error(c.call("screenshot")), "the task has ended (done)") + + def test_the_budget_and_the_turns_end_hold_when_the_screen_changes(self): + c = self.start(steps=1) + self.assertEqual(value(c.call("click", x=1, y=1)), {"steps_left": 0, "done": True}) + self.assertEqual(error(c.call("click", x=1, y=1)), "the steps are used up") + self.assertEqual(value(c.call("info"))["steps_left"], 0) + c = self.start(steps=4) + c.call("click", x=1, y=1) + self.bridge.end_turn() + self.assertEqual(self.evaluated(), 4) + self.assertEqual(error(c.call("click", x=1, y=1)), "the agent's turn has ended") + + if __name__ == "__main__": unittest.main() diff --git a/app/tests/test_standard_driver_hooks.py b/app/tests/test_standard_driver_hooks.py index 35b38e98..9dd916b0 100644 --- a/app/tests/test_standard_driver_hooks.py +++ b/app/tests/test_standard_driver_hooks.py @@ -462,3 +462,101 @@ def test_setup_gets_the_benchmarks_own_limit(self): failed = json.load(f)["exception_info"] self.assertEqual(failed["exception_type"], "EnvironmentSetupError") self.assertIn("setup: no reply in 1 s", failed["exception_message"]) + + +# A driver that is told when the agent is given its task: it marks the trial, and records each score's answer. +GIVEN_DRIVER = '''import json, os, time +from nullius.standard.jsonl import serve +def setup(unit, dir, **told): + with open(os.path.join(os.path.dirname(dir), "told.json"), "w") as f: + json.dump(told, f) + return {"prompt": "hi"} +def task_given(unit, dir): + with open(os.path.join(os.path.dirname(dir), "given"), "a") as f: + f.write(json.dumps({"unit": unit, "dir": dir}) + "\\n") + return {} +def score(unit, dir, transcript, answer): + with open(os.path.join(os.path.dirname(dir), "answer.txt"), "w") as f: + f.write(answer) + return {"reward": 1.0, "passed": True, "detail": {}} +serve({"units": lambda dataset: {"units": ["a"]}, "setup": setup, "task_given": task_given, "score": score}) +''' + + +class TaskGiven(unittest.TestCase): + """The run tells a driver that asks (driver.task_given) that the agent is being given its task, and waits for its + answer, before the first prompt of an ACP agent and before a command agent starts: a real run, the fake ACP agent + and a command of the test's.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + self._home = install.HOME + install.HOME = os.path.join(self.tmp, "home") + self.addCleanup(setattr, install, "HOME", self._home) + self.env = {"MODEL_API_URL": "http://127.0.0.1:9/v1"} + + def given(self): + return sorted(str(p) for p in Path(self.tmp).rglob("given")) + + def trial(self, other, code=GIVEN_DRIVER, **bench): + here = os.path.join(self.tmp, "bench") + os.makedirs(here, exist_ok=True) + with open(os.path.join(here, "driver.py"), "w") as f: + f.write(code) + jobs = tempfile.mkdtemp(dir=self.tmp) + job = run.run(dict(BENCH, _here=here, _kind="benchmark", **bench), other, model="m-1", provider="custom", + jobs_root=jobs, concurrency=1, env=self.env, say=lambda s: None, unsandboxed=True) + return next(os.path.join(job, d) for d in os.listdir(job) if "__u" in d) + + def test_the_flag_is_true_or_false(self): + self.assertEqual(definitions.problems(dict(BENCH, driver=dict(BENCH["driver"], task_given=True)), "benchmark"), []) + self.assertIn("driver.task_given: true or false - whether the driver is told when the agent is given its task", + definitions.problems(dict(BENCH, driver=dict(BENCH["driver"], task_given="yes")), "benchmark")) + + def test_an_acp_agent_is_sent_its_task_once_the_driver_has_taken_it(self): + seen, prompt = [], run.acp.Client.prompt + + def watched(client, *a, **kw): + seen.append(self.given()) + return prompt(client, *a, **kw) + with mock.patch.object(run.acp.Client, "prompt", watched): + trial = self.trial(definitions.get("agent", "fake", ROOT), driver=dict(BENCH["driver"], task_given=True)) + mark = os.path.join(trial, "given") + self.assertEqual(seen, [[mark]]) + with open(mark) as f: + self.assertEqual(json.loads(f.read()), {"unit": "a", "dir": os.path.join(trial, "workspace")}) + with open(os.path.join(trial, "answer.txt")) as f: + self.assertEqual(f.read(), "HI") + + def test_a_command_agent_starts_once_the_driver_has_taken_its_task(self): + counting = [sys.executable, "-c", "import pathlib, sys; print(len(list(pathlib.Path(sys.argv[1]).rglob('given'))))", + self.tmp, "{prompt}"] + command = dict(definitions.get("agent", "fake", ROOT), start={"command": counting}) + trial = self.trial(command, driver=dict(BENCH["driver"], task_given=True)) + with open(os.path.join(trial, "answer.txt")) as f: + self.assertEqual(f.read(), "1") + + def test_a_driver_that_does_not_ask_and_a_control_are_told_nothing(self): + trial = self.trial(definitions.get("agent", "fake", ROOT)) + with open(os.path.join(trial, "answer.txt")) as f: + self.assertEqual(f.read(), "HI") + controls = {"nop": {"source": "nullius", "from": "nothing"}} + control = definitions.control(dict(BENCH, controls=controls), "nop") + self.trial(control, driver=dict(BENCH["driver"], task_given=True), controls=controls) + self.assertEqual(self.given(), []) + + def test_a_driver_that_fails_it_ends_the_unit_before_the_task_is_sent(self): + seen, prompt = [], run.acp.Client.prompt + + def watched(client, *a, **kw): + seen.append(True) + return prompt(client, *a, **kw) + failing = GIVEN_DRIVER.replace(" return {}\ndef score", " raise ValueError('no window')\ndef score") + with mock.patch.object(run.acp.Client, "prompt", watched): + trial = self.trial(definitions.get("agent", "fake", ROOT), failing, driver=dict(BENCH["driver"], task_given=True)) + with open(os.path.join(trial, "result.json")) as f: + failed = json.load(f)["exception_info"] + self.assertEqual(failed["exception_type"], "EnvironmentSetupError") + self.assertIn("the benchmark was not told that the task is given", failed["exception_message"]) + self.assertEqual(seen, []) diff --git a/docs/agents/supported/README.md b/docs/agents/supported/README.md index 2903712c..0a799583 100644 --- a/docs/agents/supported/README.md +++ b/docs/agents/supported/README.md @@ -28,7 +28,7 @@ Every agent in the catalogue, the ports each version has proven, and the benchma | `agent.conversation/v1` | Conversation | Several prompts in one session, each answered in its own turn, the earlier ones remembered. | | `agent.search/v1` | Search | Search an HTTP collection, retrieve a source and answer with its evidence using the agent's available tools. | | `agent.browser/v1` | Browser | A browser reached through an MCP server Nullius starts. | -| `agent.computer/v1` | Computer use | A desktop the agent sees in screenshots and drives with mouse and keyboard actions. | +| `agent.computer/v1` | Computer use | A desktop the agent sees in screenshots and drives with mouse and keyboard actions, reached through steps alone: what the agent learns of it comes in a step's answer. | diff --git a/docs/benchmarks/supported/osworld1.md b/docs/benchmarks/supported/osworld1.md index cf9ac1b3..00426bf9 100644 --- a/docs/benchmarks/supported/osworld1.md +++ b/docs/benchmarks/supported/osworld1.md @@ -49,7 +49,7 @@ Open-ended tasks on a real Ubuntu desktop and its applications, each in a VM of ## Good to know -- An agent runs it through the computer port: the agent's tool calls drive OSWorld's own loop, one step for each acting call. Steps (100) and Pause (3.0 seconds) set that loop. +- An agent runs it through the computer port: the agent's tool calls drive OSWorld's own loop, one step for each acting call. A step's observation is the screenshot that loop hands its agent for the next decision, with no capture added; the files a step names are read once its actions and their pauses have run. Steps (100) and Pause (3.0 seconds) set that loop. - Original runs one of OSWorld's own agents, chosen by the Agent option (M3 by default), with a model it is published with. - Providers: OpenAI for OpenAI CUA; OpenAI-compatible for Muse Spark (Meta's API) and for EvoCUA, AutoGLM-OS and Mano (your model server); OpenRouter for the others. - Only M3, PromptAgent and Qwen-CUA have completed tasks through Nullius. The other agents were checked up to their first model call. diff --git a/docs/ports/agent-computer-v1.md b/docs/ports/agent-computer-v1.md index 8201438c..f0d8e3bd 100644 --- a/docs/ports/agent-computer-v1.md +++ b/docs/ports/agent-computer-v1.md @@ -1,22 +1,28 @@ # The computer port: `agent.computer/v1` -A desktop the agent sees in screenshots and drives with pointer and keyboard actions. A desktop benchmark's runner serves one MCP server, `computer`, to the agent's session; its tools are the desktop's basic operations. The agent is a client alone. An agent with a driver of its own installs it with `upload` and `python` and sends its actions with `step`. Any other agent uses the pointer and keyboard tools. Every agent gets the same tool list. +A desktop the agent sees in screenshots and drives with pointer and keyboard actions. A desktop benchmark's runner serves one MCP server, `computer`, to the agent's session; its tools are the desktop's basic operations. The agent is a client alone. Once the agent is given its task, it reaches the desktop through acting calls alone, and everything it learns of the desktop comes in an acting call's answer. One acting call is one step. An agent with a driver of its own uploads it and starts it with `prepare` before the task, and talks to it through a step's actions and files. Every agent gets the same tool list. Coordinates are pixels of the screen, from its top left corner. A result's text is JSON. +## The window before the task + +| Rule | Detail | +| --- | --- | +| Opens | When the agent's session opens | +| Closes | When the run sends the agent its task: Nullius tells the benchmark first, and the task waits until the window is closed. A session that starts with its task already given, such as a command that takes the prompt, has no window | +| In it | `upload` and `prepare`, as often as needed, neither a step; `info` and `screenshot`. An acting call is refused and takes no step | +| After it | `upload` and `prepare` are refused, with words that say the task has been given | + ## Reading None of these is a step. | Tool | Arguments | Result | | --- | --- | --- | -| `info` | none | `{screen: [width, height], steps, steps_left, pause, python_seconds, password}` | -| `screenshot` | none | An image: the screen as PNG, at its own size | -| `read` | `path`, in the desktop | `{data}`: base64, or null when there is no such file. A file over 12 MiB is an error | -| `python` | `code` | The desktop's reply, `{status, output, error, returncode}`; an error when it gave none within `python_seconds` | -| `upload` | `path`, in the desktop; `data`, base64, at most 4 MiB a call; `append`, false (the default) to start the file and true to add to it; `sha256`, optional, the whole file's. With `sha256` and no `data`, the call takes the run's file of that `sha256` and starts the file; `append` is then refused | `{size}`: the file's size so far | - -`python` runs as the desktop's user, in the desktop's Python. It is for reading the desktop and for the agent's own programs there. A reply whose code failed, with its `returncode`, is the tool's result, as a shell's would be. +| `info` | none | `{screen: [width, height], steps, steps_left, pause, password, folder}`. `folder`: a folder of the desktop that is the agent's own, made empty for the unit | +| `screenshot` | none | The observation of the last step, as PNG. Before the first step, the picture the benchmark took at the start, before the window. It takes no new picture: two calls with no step between them return the same bytes | +| `upload` | `path`, inside `folder` (absolute, or relative to it, with no `.` or `..` part); `data`, base64, at most 4 MiB a call; `append`, false (the default) to start the file and true to add to it; `sha256`, optional, the whole file's. With `sha256` and no `data`, the call takes the run's file of that `sha256` and starts the file; `append` is then refused | `{size}`: the file's size so far. In the window alone | +| `prepare` | `actions`: a list of texts, each Python run in the desktop as a step's action is, with no pause after it (`WAIT` does nothing); `files`: as for `step` | `{files: {PATH: BASE64 or null}}`, once the actions have run. In the window alone; an action the desktop does not end in time is an error and stops it | ## Acting @@ -30,12 +36,22 @@ One call is one step. Points use `x` and `y` in screenshot pixels at the screen' | `scroll` | `x`, `y`; `dy` and `dx`, each from -100 to 100 and default 0, in wheel clicks: `dy` down, `dx` right, negative the other way; at least one is not 0 | The wheel turned at the point | | `type` | `text` | The text typed key by key | | `key` | `keys`: a list pressed together, such as `["ctrl", "s"]` | The chord | -| `wait` | none | No action, the pause alone | -| `step` | `actions`: a list of texts, each Python run in the desktop with `pyautogui` and `time` imported, or `WAIT`; `response`, the agent's words, optional; `screenshot`, default false; `end`, optional: `done` or `fail` | The actions in order, as one step. An empty list is a step with no action. With `end`, the task then ends in that step, as by `done` or `fail` | +| `wait` | none | No action, the pause alone: the way to look again, one step and a fresh observation | +| `step` | `actions`: a list of texts, each Python run in the desktop with `pyautogui` and `time` imported, or `WAIT`; `response`, the agent's words, optional; `screenshot`, default false; `end`, optional: `done` or `fail`; `files`, optional: any number of paths inside `folder` | The actions in order, as one step. An empty list is a step with no action. With `end`, the task then ends in that step, as by `done` or `fail` | | `done` | `message`, optional | The task is complete | | `fail` | `reason`, optional | The task cannot be done | -A call of `click`, `move`, `drag`, `scroll`, `type`, `key`, `wait` or `step` answers once its actions have run and their pauses have passed, with `{steps_left, done}`; `done` is true once the task takes no more calls. Each adds the screen as a PNG image; `step` adds it only when `screenshot` is true. `done` and `fail` return `{steps_left, done}` with no image as the benchmark takes them, before it runs them: the benchmark's loop asks for nothing after them. A `step` with `end` answers once its actions have run, with `done` true, and the benchmark then takes its end. `DONE` or `FAIL` among the actions is refused. +A call of `click`, `move`, `drag`, `scroll`, `type`, `key`, `wait` or `step` answers once its actions have run and their pauses have passed, with `{steps_left, done}`; `done` is true once the task takes no more calls. Each adds the step's observation as a PNG image, the bytes `screenshot` returns afterwards; `step` adds it only when `screenshot` is true. `done` and `fail` return `{steps_left, done}` with no image as the benchmark takes them, before it runs them: the benchmark's loop asks for nothing after them. A `step` with `end` answers once its actions have run, with `done` true, and the benchmark then takes its end. `DONE` or `FAIL` among the actions is refused. + +A step's `files` are read once its actions have run and their pauses have passed: the answer is `{steps_left, done, files: {PATH: BASE64 or null}}`, null for a file that is not there. A file over 12 MiB, or one the desktop cannot read, is `{PATH: {error}}` for that file alone; the step has run all the same. A path outside `folder` refuses the call at once: it takes no step and never reaches the desktop. A whole answer is at most 24 MiB: the files are taken in their order while the answer holds them, with its image, and each file that would take it past 24 MiB is `{PATH: {error}}`. The same holds for `prepare`. + +## The observation + +| Rule | Detail | +| --- | --- | +| When | Fixed as a step that ran an action ends. A step that ran no action leaves it as it was | +| Which | The benchmark's own: for a loop that hands its agent an observation, such as [OSWorld1](../benchmarks/supported/osworld1.md)'s, the screenshot it hands for the next decision, with no capture added. For a loop that hands none, one picture taken once the step's last pause has passed | +| Between two steps | Nothing else of the desktop is shown. What `prepare` changed on the screen shows in the first step's observation | ## Rules @@ -43,35 +59,43 @@ A call of `click`, `move`, `drag`, `scroll`, `type`, `key`, `wait` or `step` ans | --- | --- | | A step | One call of an acting tool; the benchmark's own loop counts these calls | | Calls | Each call is checked against its tool's schema in `tools/list`, an upload's base64 decoded, before it waits for the desktop. A schema lists every argument its tool takes, and its patterns read alike in any engine. A call that does not hold is refused at once: it takes no step and never reaches the desktop. A message is JSON-RPC 2.0, one a line, at most 24 MiB with its newline; `NaN` and `Infinity` are no JSON, and any other message is refused. Every answer is at most 24 MiB too | -| The limits | In `initialize`'s `instructions`, in `info`, and as `steps_left` in every acting result | +| The limits | `steps`, `steps_left` and `pause`, in `initialize`'s `instructions`, in `info`, and as `steps_left` in every acting result. A `prepare` action is bounded as a step's action is | | The step budget | [OSWorld1's `max_steps`](../benchmarks/supported/osworld1.md#options). The benchmark's loop stops at its budget. After the last step, every tool but `info` answers with an error, and the benchmark evaluates the desktop as it is. An acting call sent while every step is taken is refused at once | | Model calls | A benchmark may bound the agent's model calls at its step budget | | The turn's end | A turn that ends with neither `done` nor `fail` ends the task, and the benchmark evaluates the desktop as it is. The driver tells the runner in one message. That message ends the wait for the agent's next call. The actions of a step not yet run are not run, and that call is refused, its `end` with it. A `done` or `fail` taken before the message still reaches the benchmark. Each step left then passes with no action. The benchmark's loop counts those steps up to its budget, and its own records list no action for them. The call log holds the agent's own steps | -| Input | Pointer and keyboard input goes through the acting tools. Input sent through `python` is outside the port | +| Input | Pointer and keyboard input goes through the acting tools. Input sent by `prepare` is outside the port | | One at a time | The server runs one desktop operation at a time, in the order the calls arrive. A call sent during a step waits for it | | Before the desktop is ready | `initialize`, `tools/list` and `info` answer at once; any other call waits. The session starts once the desktop is ready | | Errors | An error (`isError`) is a call refused, or a call the desktop could not run or did not answer in time; it has the desktop's words. The agent may call again | | Uploads | By content: `upload` takes the bytes, in base64. A file sent once for a run: when `upload` names the file's `sha256` and the run already holds that content, the runner puts it into this unit's desktop at that call, and `data` may be left out. The run keeps a file once chunks sent from its start, in one unit, hash to the `sha256` given with the first, up to 256 MiB. Those chunks are the unit's own: they are dropped when the file starts again and at the unit's end. Nothing else passes from one unit to the next | | Connections | The server accepts every connection of the unit's session and keeps the task's steps and waiting calls outside them. A session may connect again; the task goes on when a connection closes during a step. The turn's end ends the task: Nullius sends it when the agent's process is gone or its time is up. Each connection's answers are written apart: a client that stops reading holds up no other call, and once 48 MiB of its answers wait, its connection is closed. A connection whose input has ended is closed once its calls are answered | -| The record | Every tool call goes to the unit's record as one line, `{tool, start, end, isError}`, beside the benchmark's own records; an answer over 24 MiB is an error. No argument and no result is recorded | +| The record | Every tool call goes to the unit's record as one line, `{tool, start, end, isError}`, beside the benchmark's own records; an answer over 24 MiB is an error. No argument and no result is recorded. The benchmark's records list every action's text: a step's actions pass through its loop; a `prepare` call's go to a record of their own, `{start, actions, files}` per call, written before they run | | More tools | A benchmark may add tools, such as a question to its user. The agent finds them in `tools/list` | +## Known limits + +- Inside the desktop, the agent's code can read and write wherever the desktop's user can and can create links outside `folder`; `folder` bounds the names of paths passed to `upload` and `files`, and the benchmark's records hold the code of every step action and `prepare` call. +- An action's code may start a process that goes on after its step, as an OSWorld action may. The agent learns what it did from a later step's answer alone. +- In the window the desktop already holds the task's files and windows. An agent that guessed its task from them could work there; the benchmark's records list every `prepare` call's code. +- An agent that is not sandboxed (the local backend) could reach the machine beside the port. A comparison that has to be trusted runs on the container backend. + ## For a benchmark's runner The shared code is `app/nullius/standard/computer_port.py`, with the standard library alone, and `computer_pipe.py` beside it. | Piece | What the runner gives or does | | --- | --- | -| `Desktop` | Five methods over its desktop: `python(code)`, `upload(path, data)`, `read(path)`, `capture()`, `password()` | -| `Bridge` | Handed to the benchmark's own loop as its agent: `reset` does nothing, and each `predict` returns the actions of the agent's next acting call. Given the loop's step budget and pause, the screen's size and `python_seconds`; `finish` once the loop has returned | +| `Desktop` | Five methods over its desktop: `python(code)`, `upload(path, data)`, `read(path)`, `capture()`, `password()`; and optionally `observation()`: the screenshot the benchmark's loop took as the step it just ran ended. Without it, the bridge takes one by `capture()` | +| `Bridge` | Handed to the benchmark's own loop as its agent: `reset` does nothing, and each `predict` returns the actions of the agent's next acting call. Given the loop's step budget and pause, the screen's size and `folder`, an absolute path in the desktop that the bridge makes empty at the loop's first `predict`; `window=True` when the benchmark's driver is told the task is given (else the session has no window); `prepared`, the file of the `prepare` record; `finish` once the loop has returned | | `Server` | The MCP server at a Unix socket in a folder of the user's alone, with the path of the call log | | `Held` | The run's folder for files sent once, when the runner keeps one. A file is kept through a temporary file of its own, and read only when it is no link | | The session's server | Setup's `mcp` entry runs `computer_pipe.py SOCKET`, which joins the session's stdio to the socket | -| `ready`, `lifeline`, `Stop` | The ready file, written at the loop's first `predict`; the lifeline from the driver, whose line `turn-end` is the turn's end and whose close is a stop; the one stop, whose last resort gives its own callback half a second before it exits | +| `ready`, `lifeline`, `Stop` | The ready file, written at the loop's first `predict`; the lifeline from the driver, whose line `turn-end` is the turn's end, whose line `task-given` closes the window, and whose close is a stop; the one stop, whose last resort gives its own callback half a second before it exits | +| The window's end | `driver.task_given` in the definition: Nullius sends `task_given` before the agent's task, and the driver closes the window (`Bridge.close_window`, or the lifeline's line `task-given`) before it answers | ## The kit -`cli conform AGENT --port agent.computer/v1` runs the kit `agent-computer-v1`: eight checks, each on a desktop of its own, reached through this port's server. A program decides each check from the desktop's own records and the server's call log. +`cli conform AGENT --port agent.computer/v1` runs the kit `agent-computer-v1`: nine checks, each on a desktop of its own, reached through this port's server. The run tells the kit when it gives the agent its task; the window before it is open until then. A program decides each check from the desktop's own records and the server's call log. | Check | Kind | The task | Passes when | | --- | --- | --- | --- | @@ -83,16 +107,17 @@ The shared code is `app/nullius/standard/computer_port.py`, with the standard li | `scroll` | skill | Scroll to the end of a list and report the large words there in order | The final message contains an uninterrupted sequence of the drawn words among at most twelve vocabulary occurrences, and the list moved | | `recover` | skill | Click a named button; the first acting call, whatever it is, is answered with an error and not run | An acting call sent after the error's answer succeeds, and the button is clicked as in `click` | | `batch` | skill | Fill three text fields with a budget of two steps | Each field holds its word | +| `late` | skill | A button appears a few seconds after the task is given, away from the screen's centre at a place drawn for the unit; click it once | The desktop logged a left press on the button with every input inside an acting call. A picture was taken at a step's end after the button appeared and before the action containing the press began. | -`see` shows three distinct common nouns drawn at 63 pixels tall, with gaps between words. `scroll` shows two at the list's end, below its initial view, at 56 pixels tall. Both sample from 96 everyday nouns chosen to avoid screen, desktop and action vocabulary and the fixture's labels. The verdict reads whole vocabulary words in the final message, ignoring case. The drawn words must occur together in the shown order at least once, with no other vocabulary word inside that sequence. Prose and punctuation may separate them. Repeated answers and vocabulary words before or after the sequence are accepted, up to twelve vocabulary occurrences in the whole message, counting repeats. `type` tests typing in a field that already has the focus; no pointing is needed. `limit` needs at least one button pressed within the two-step budget, no input after the last step and no acting call once the steps are used up. +`see` shows three distinct common nouns drawn at 63 pixels tall, with gaps between words. `scroll` shows two at the list's end, below its initial view, at 56 pixels tall. Both sample from 96 everyday nouns chosen to avoid screen, desktop and action vocabulary and the fixture's labels. The verdict reads whole vocabulary words in the final message, ignoring case. The drawn words must occur together in the shown order at least once, with no other vocabulary word inside that sequence. Prose and punctuation may separate them. Repeated answers and vocabulary words before or after the sequence are accepted, up to twelve vocabulary occurrences in the whole message, counting repeats. `type` tests typing in a field that already has the focus; no pointing is needed. `limit` needs at least one button pressed within the two-step budget, no input after the last step and no acting call once the steps are used up. `late` draws its button 3 s after the task is given, never over the screen's centre: a click there without looking does not find it. -Every check also fails when a pointer or key event came outside an acting call: input sent through `python` fails it. Each check has 8 steps (2 for `limit` and `batch`), a pause of 1 s, 90 s for one `python` call and 600 s for the agent's turn. +Every check also fails when a pointer or key event came outside an acting call: input sent by `prepare` fails it. Each check has 8 steps (2 for `limit` and `batch`), a pause of 1 s, 90 s for one action and 600 s for the agent's turn. | Piece | Where | | --- | --- | -| The desktop | A container of the kit's image, `desktop.Dockerfile` on Nullius's base image, built the first time a unit needs it: an X display at 1920 by 1080, a window manager, a session bus, the accessibility bus, Python with Tk, `pyautogui` and Pillow. The agent's Python and its actions run as its user; the kit's fixture runs as another, its log out of that user's reach | +| The desktop | A container of the kit's image, `desktop.Dockerfile` on Nullius's base image, built the first time a unit needs it: an X display at 1920 by 1080, a window manager, a session bus, the accessibility bus, Python with Tk, `pyautogui` and Pillow. The agent's actions run as its user; the kit's fixture runs as another, its log out of that user's reach | | The server and the loop | In the kit's driver: the port's server over the unit's desktop, and a loop of the kit's own that runs each step's actions and waits the pause | | What a stop leaves | A unit that never reaches its score is let go by the next unit or by the driver's end; a reaper the driver starts removes the containers and sockets of a driver that was killed | -| The controls | The null control passes no check; the reference makes the right calls through the unit's own server, reads the words and their gaps from the screenshot it gets, and passes all eight | +| The controls | The null control passes no check; the reference is given its task as it starts, makes the right calls through the unit's own server, reads the words and their gaps from the screenshot it gets and, for `late`, looks again by `wait` until a step's observation was taken with the button up, then clicks it: it passes all nine | A machine that runs the kit needs Docker, or Podman 4.7 or later, as the container backend does. The first unit builds the image (about 0.5 GB over the base image). diff --git a/docs/standard.md b/docs/standard.md index 0c0cd9f1..6c6b3fcb 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -31,7 +31,7 @@ A pair is an **agent benchmark** × an **agent**, or a **capability benchmark** | `agent.conversation/v1` | Several prompts in one session, the earlier turns remembered | Turn 2 asks for what turn 1 said | | `agent.search/v1` | Search an HTTP collection with the agent's own tools, retrieve a source and answer with evidence | Find a source, choose the current one, follow a reference (3 checks) | | `agent.browser/v1` | A browser, through an MCP server Nullius starts | Defined; its kit comes with the first browser benchmark | -| `agent.computer/v1` | A desktop seen in screenshots and driven by mouse and keyboard actions, through [the tools of an MCP server the benchmark's runner serves](ports/agent-computer-v1.md) | Four basic checks, four skills on a small desktop in a container ([the kit](ports/agent-computer-v1.md#the-kit)) | +| `agent.computer/v1` | A desktop seen in screenshots and driven by mouse and keyboard actions, through [the tools of an MCP server the benchmark's runner serves](ports/agent-computer-v1.md), reached through steps alone once the task is given: each step's answer holds what the agent learns of it | Four basic checks, five skills on a small desktop in a container ([the kit](ports/agent-computer-v1.md#the-kit)) | | `cap.memory/v1` | `add {session}` then `recall {question}` → `{memory}`; a session is `{id, date, turns: [{role, text}], text}` | Recall a place and an exact name among distractors (2 checks); nothing it stores is a credential | - **The last message** follows ACP `messageId` boundaries and other events (a tool call, a notice). Without IDs, session metadata parts messages but never erases the final answer. @@ -58,7 +58,7 @@ no generic agent-plus-capability recipe or swap control. | Rule | Detail | | --- | --- | -| Declared | A kit's `kit_version`: 1 for every kit but the tools kit, at 2 | +| Declared | A kit's `kit_version`: 1 for every kit but the tools kit and the computer kit, at 2 | | Bumped | When a check gets stronger or tests something new; rewording a check that still tests the same thing keeps the version and the proofs made on it | | Recorded | `kit_version` in the `proven` entry; none means version 1 | | Older proofs | Still proofs, read as *proven on kit v1; kit v2 is current* | @@ -436,6 +436,7 @@ use the source folders and repository checks. | `score` | `unit, dir, transcript, answer`, `setup`, `verified` for an own image, `judge: {model, base_url, key}` when judged | `{reward, detail}`, and `passed` when the benchmark passes or fails; `parts: {n, by, parts}` when the reward is a mean over several parts (a conversation's questions), so the job's mean weighs it by `n` and breaks it down by `by`; `stopped`, the reason of a stop asked while the unit ran, kept as `verifier_result.stopped` | | `oracle` | `{unit, dir}` | `{answer}`, the reference; a workspace benchmark also writes it into `dir` | | `end` | `{unit, dir}`, with `driver.unit_end`, once a unit an agent ran has ended, also after a stop, when its `setup` was sent and the driver still runs with no request left unanswered | `{}` | +| `task_given` | `{unit, dir}`, with `driver.task_given`, just before an agent is sent its task: its first prompt, or its command | `{}` once the benchmark's side has taken it; a failure ends the unit before the task is sent | An `answer` or `score` error that begins `EnvironmentSetupError: `, `VerifierError: `, `CancelledError: ` or `CauseUnknownError: ` keeps that kind. @@ -452,6 +453,7 @@ An `answer` or `score` error that begins `EnvironmentSetupError: `, `VerifierErr | `limits.call_seconds` | How long the relay waits for one model call's reply (default 900). A benchmark whose agent waits longer for a call sets it above that agent's own wait, so the agent's client, not the relay, ends a long call | | `driver.participant` | `true`: `setup` is also sent `participant`, the kind of what runs the unit, for a driver that starts something only an agent uses. A driver whose `setup` takes `unit` and `dir` alone leaves it out | | `driver.unit_end` | `true`: the driver is sent `end` (`unit`, `dir`) once each unit an agent ran has ended, scored or not, so that what its setup started for the agent ends with the unit. Only a unit whose `setup` was sent gets it, and only while that driver still runs: a driver closed after a request got no reply in its time is not asked, nor is one whose request a stop left unanswered. A control's or a setup's unit gets none, nor does any other driver | +| `driver.task_given` | `true`: the driver is sent `task_given` before each agent is given its task, and the task waits for its answer: [the computer port](ports/agent-computer-v1.md#the-window-before-the-task) closes its window before the task there. A control's or a setup's unit gets none | | `limits.attempts` | How the upstream tells attempts apart; the driver is then sent `attempt` with `setup`, `turn` and `answer` | | `units.redistribute` | `false` when the data may not be redistributed: `cli prove` then records no unit id |
AgentPortsTools, the skills of agent.tools/v1
One taskWorkspace filesMCP toolsConversationSearchComputer usedecoyschainretry