Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions app/nullius/catalogue/agents/codex/agent.json
Original file line number Diff line number Diff line change
Expand Up @@ -141,13 +141,13 @@
},
"agent.computer/v1": {
"passed": "4/4",
"on": "2026-11-17",
"on": "2026-11-19",
"model": "gpt-6-luna",
"provider": "openai",
"version": "2.2.2",
"kit_version": 1,
"kit_version": 2,
"backend": "container",
"job": "2026-11-17__12-53-08__kit__agent-computer-v1__codex-2.2.2",
"job": "2026-11-19__09-44-46__kit__agent-computer-v1__codex-2.2.2",
"repeat": 3,
"checks": {
"see": "3/3",
Expand All @@ -171,6 +171,10 @@
"batch": {
"passed": "3/3",
"ok": true
},
"late": {
"passed": "3/3",
"ok": true
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion app/nullius/catalogue/benchmarks/osworld1/benchmark.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
],
"origin": "official",
"good_to_know": [
"An agent runs it through the computer port: the agent's tool calls drive OSWorld's own loop, one step for each acting call. Steps (100) and Pause (3.0 seconds) set that loop.",
"An agent runs it through the computer port: the agent's tool calls drive OSWorld's own loop, one step for each acting call. A step's observation is the screenshot that loop hands its agent for the next decision, with no capture added; the files a step names are read once its actions and their pauses have run. Steps (100) and Pause (3.0 seconds) set that loop.",
"Original runs one of OSWorld's own agents, chosen by the Agent option (M3 by default), with a model it is published with.",
"Providers: OpenAI for OpenAI CUA; OpenAI-compatible for Muse Spark (Meta's API) and for EvoCUA, AutoGLM-OS and Mano (your model server); OpenRouter for the others.",
"Only M3, PromptAgent and Qwen-CUA have completed tasks through Nullius. The other agents were checked up to their first model call.",
Expand Down
47 changes: 38 additions & 9 deletions app/nullius/catalogue/benchmarks/osworld1/driver.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@
the MCP server computer (the port's pipe, joined to the runner's
socket) and the agent's time. The options max_steps and pause are
the loop's
task_given - an agent's unit (driver.task_given): the agent is about to be
given its task. The runner is told on its lifeline, closes the
port's window before the task and says so; this answers then
answer - a setup's (original, model): the script of the run's agent option
(agents.json: the upstream's scripts/python/run_*.py) as published,
on the one task (a test_all_meta file naming it), with the arguments
Expand All @@ -40,8 +43,8 @@
waits for the runner to end and reads result.txt from the runner's
folder, never the workspace, and only from a runner this driver
started that ended by itself with status 0, through folders opened
without following a link. It copies result.txt and the call log to
the trial's verifier/osworld/.
without following a link. It copies result.txt, the call log and the
record of prepare calls to the trial's verifier/osworld/.
For the null control, nop.py: the upstream's run_single_example
with an agent that answers DONE at its first step, its result.txt
read the same way
Expand Down Expand Up @@ -120,6 +123,7 @@
REMOVE_SECONDS = 30 # the docker commands that remove a unit's VMs, together
FALLBACK_SECONDS = 2 # this driver's removal when no reaper could start: inside the core's 3 s
RUNNERS = {} # the runner folder of each unit this driver set up -> its process
GIVEN_SECONDS = 30 # the runner's word that the window before the task is closed


def _load(path):
Expand Down Expand Up @@ -473,7 +477,9 @@ def _start_runner(unit, dir, instruction, until):
argv = [sys.executable, RUNNER_FILE, "--example", task, "--result-dir", os.path.join(where, RESULTS),
"--socket", os.path.join(sock_dir, "s"), "--ready", ready, "--lifeline", str(life_r),
"--steps", "100" if steps == "default" else steps, "--pause", str(_option("pause", "3.0", float)),
"--vm", vm, "--calls", os.path.join(where, "computer-calls.jsonl"), "--held", held]
"--vm", vm, "--calls", os.path.join(where, "computer-calls.jsonl"), "--held", held,
"--prepared", os.path.join(where, "computer-prepare.jsonl"),
"--given", os.path.join(sock_dir, "given.json")]
lock = open(START_LOCK, "a")
while True: # VM starts one after another, as the setups' do
try:
Expand Down Expand Up @@ -575,6 +581,26 @@ def _turn_end(entry):
pass


def task_given(unit, dir, **_):
"""The agent is about to be given its task: the runner closes the port's window before it and says so in the
socket's folder; this answers once it has, within GIVEN_SECONDS."""
entry = RUNNERS.get(_runner_dir(dir))
if entry is None or entry.get("life") is None:
raise RuntimeError("no runner of this driver's for this unit: the window before the task cannot be closed")
said = os.path.join(entry["sock_dir"], "given.json")
os.write(entry["life"], _port().TASK_GIVEN + b"\n")
t0 = time.time()
while not os.path.exists(said):
if _exited(entry["p"]) is not None or time.time() - t0 > GIVEN_SECONDS or CLOSED.wait(0.05):
raise RuntimeError("the runner did not say that the window before the task is closed")
with open(said, encoding="utf-8") as f:
ok = json.load(f).get("ok")
os.unlink(said)
if not ok:
raise RuntimeError("the runner could not close the window before the task: " + str(ok))
return {}


def _reap_runner(where, seconds=None):
"""The unit's runner let go, given ``seconds`` (REAP_SECONDS) to end by itself, its whole group killed while its
leader is still unreaped, then reaped; the VM containers of the unit's image link removed within REMOVE_SECONDS,
Expand Down Expand Up @@ -772,13 +798,16 @@ def _agent_score(where, dir, domain, eid):
value = float(result.decode().strip())
except (OSError, ValueError, RuntimeError) as e:
raise RuntimeError(f"the runner's result could not be read ({e}): " + _tail(os.path.join(where, "runner.log")))
try:
calls = _plain(trial, ".work", "osworld", "computer-calls.jsonl")
except (OSError, RuntimeError):
calls = b""
kept = {"computer-calls.jsonl": b""}
for name in ("computer-calls.jsonl", "computer-prepare.jsonl"): # the record of prepare calls when there is one
try:
kept[name] = _plain(trial, ".work", "osworld", name)
except (OSError, RuntimeError):
pass
calls = kept["computer-calls.jsonl"]
out = os.path.join(trial, "verifier", "osworld")
os.makedirs(out, exist_ok=True)
for name, data in (("result.txt", result), ("computer-calls.jsonl", calls)):
for name, data in [("result.txt", result)] + sorted(kept.items()):
with open(os.path.join(out, name), "wb") as f:
f.write(data)
return {"reward": value, "detail": {"result": value, "by": "runner.py, run_single_example, DesktopEnv.evaluate",
Expand Down Expand Up @@ -816,7 +845,7 @@ def score(unit, dir, transcript=None, answer=None, setup=None, **_):
return _scored(where, "nop.py, DesktopEnv.evaluate")


HANDLERS = {"units": units, "setup": setup, "answer": answer, "score": score, "end": end}
HANDLERS = {"units": units, "setup": setup, "task_given": task_given, "answer": answer, "score": score, "end": end}


def main():
Expand Down
68 changes: 53 additions & 15 deletions app/nullius/catalogue/benchmarks/osworld1/runner.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,24 @@
"""One unit of an agent on OSWorld1: the upstream's DesktopEnv and its own run_single_example, imported and called as
they are, with the computer port's bridge as the loop's agent (nullius.standard.computer_port).
they are, with the computer port's bridge as the loop's agent (nullius.standard.computer_port, agent.computer/v1).

runner.py --example TASK_JSON --result-dir DIR --socket PATH --ready FILE --lifeline FD --steps N --pause SECONDS
--vm VM_IMAGE --calls FILE [--held FOLDER]
--vm VM_IMAGE --calls FILE [--held FOLDER] [--prepared FILE] [--given FILE]

The driver starts it in an agent's unit's setup, in the unit's runner folder (the trial's .work/osworld/), with the
upstream's folder and the nullius package on its import path. It builds the upstream's DesktopEnv as the null control
does (nop.py), serves the MCP server computer at PATH (a Unix socket in a folder of the user's alone, which the driver
made) over a Desktop of five methods on the upstream's controllers, and calls run_single_example with the bridge as the
loop's agent, N as max_steps and SECONDS as sleep_after_execution. No file of the upstream is edited, patched or
replaced.
replaced. The loop is handed the DesktopEnv through a view (Watched) that keeps the screenshot of the observation each
env.step returns: a step's observation is the one the loop hands its agent for the next decision, with no capture
added. The agent's folder in the VM is FOLDER.

- READY, a file of mode 0600: {"ok": true, "password": the desktop user's} once the loop asks for the agent's first
step (after the task's setup and the upstream's own wait); {"ok": false, "error": ...} when the unit fails before.
- FD, the read end of a pipe from the driver: a line turn-end ends the agent's turn (the loop then gets no action for
the steps left and evaluates the desktop as it is); its close stops the runner as SIGTERM does.
- FILE, the call log: one line per tool call, {tool, start, end, isError}.
the steps left and evaluates the desktop as it is); a line task-given closes the port's window before the task, and
the runner then writes --given's file, {"ok": true}; its close stops the runner as SIGTERM does.
- FILE, the call log: one line per tool call, {tool, start, end, isError}; --prepared's, one line per prepare call.
- FOLDER, the run's folder of files sent once, when the driver gives one (computer_port.Held).

A stop is taken once. The clean-up (the bridge's end and the server's close, then the desktop's close within
Expand All @@ -33,17 +36,21 @@
import threading

SCREEN = (1920, 1080)
PYTHON_SECONDS = 120 # the upstream controller's wait on a command's reply
FINISH_SECONDS = 5 # the bridge's end and the server's close
CLOSE_SECONDS = 25 # the desktop's close: the provider stops and removes its container
STOP_SECONDS = FINISH_SECONDS + CLOSE_SECONDS
FOLDER = "/tmp/agent" # the agent's own folder in the VM


class Desktop:
"""The port's five operations over the upstream's own controllers."""
"""The port's five operations over the upstream's own controllers, and the observation of the step the loop just
ran (``shot``, kept by Watched)."""

def __init__(self, env, scratch):
self.env, self.scratch = env, scratch
self.env, self.scratch, self.shot = env, scratch, None

def observation(self):
return self.shot

def python(self, code):
return self.env.controller.execute_python_command(code)
Expand All @@ -58,7 +65,7 @@ def upload(self, path, data):
os.unlink(local)

def read(self, path):
return self.env.controller.get_file(path)
return self.env.controller.get_file(path, strict=True) # a missing file at once, with no retry

def capture(self):
png = self.env.controller.get_screenshot()
Expand All @@ -70,14 +77,42 @@ def password(self):
return self.env.client_password


class Watched:
"""The upstream's DesktopEnv as its loop is handed it: the env itself for every attribute, each env.step's
observation's screenshot also kept in the Desktop, for the bridge."""

def __init__(self, env, desktop):
object.__setattr__(self, "_env", env)
object.__setattr__(self, "_desktop", desktop)

def step(self, *args, **kwargs):
out = self._env.step(*args, **kwargs)
obs = out[0] if isinstance(out, tuple) and out else None
self._desktop.shot = obs.get("screenshot") if isinstance(obs, dict) else None
return out

def __getattr__(self, name):
return getattr(self._env, name)

def __setattr__(self, name, value):
setattr(self._env, name, value)


class Turn:
"""What the lifeline's turn-end reaches: the bridge, once it exists."""
bridge = None
"""What the lifeline reaches: the bridge, once it exists, and the file that says the window before the task is
closed (``given``, written by ``said``)."""
bridge = given = said = None

def end_turn(self):
if self.bridge is not None:
self.bridge.end_turn()

def close_window(self):
if self.bridge is not None:
self.bridge.close_window()
if self.given and self.said:
self.said(self.given, ok=self.bridge is not None)


def close_within(env, seconds):
"""The desktop closed in a thread of its own, waited for ``seconds``: True when it closed in that time."""
Expand All @@ -97,13 +132,15 @@ def main(argv=None):
p = argparse.ArgumentParser()
for name in ("--example", "--result-dir", "--socket", "--ready", "--vm", "--calls"):
p.add_argument(name, required=True)
p.add_argument("--held")
for name in ("--held", "--prepared", "--given"):
p.add_argument(name)
p.add_argument("--lifeline", type=int, required=True)
p.add_argument("--steps", type=int, required=True)
p.add_argument("--pause", type=float, required=True)
a = p.parse_args(argv)
from nullius.standard import computer_port as port
stop, turn = port.Stop(STOP_SECONDS), Turn()
turn.given, turn.said = a.given, port.ready
try: # the clean-up holds before the lifeline can stop the runner
stop.install()
threading.Thread(target=port.lifeline, args=(a.lifeline, turn), name="lifeline", daemon=True).start()
Expand Down Expand Up @@ -139,11 +176,12 @@ def first(): # the loop asks for the agent's
headless=True, os_type="Ubuntu", require_a11y_tree=False, client_password="password",
enable_proxy=False)
os.makedirs(where, exist_ok=True)
bridge = port.Bridge(Desktop(env, os.path.dirname(os.path.abspath(a.calls))), a.steps, a.pause, SCREEN,
PYTHON_SECONDS, on_first=first, held=port.Held(a.held) if a.held else None)
desktop = Desktop(env, os.path.dirname(os.path.abspath(a.calls)))
bridge = port.Bridge(desktop, a.steps, a.pause, SCREEN, FOLDER, on_first=first,
held=port.Held(a.held) if a.held else None, window=bool(a.given), prepared=a.prepared)
server = port.Server(bridge, a.socket, a.calls)
turn.bridge = bridge
lib_run_single.run_single_example(bridge, env, example, a.steps, example["instruction"],
lib_run_single.run_single_example(bridge, Watched(env, desktop), example, a.steps, example["instruction"],
argparse.Namespace(sleep_after_execution=a.pause, result_dir=a.result_dir),
where, [])
except SystemExit: # SIGTERM or the lifeline's end: the unit was stopped
Expand Down
3 changes: 2 additions & 1 deletion app/nullius/catalogue/benchmarks/osworld1/standard.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@
"locked": true
},
"participant": true,
"unit_end": true
"unit_end": true,
"task_given": true
},
"mcp_servers": {
"computer": {
Expand Down
Loading
Loading