From c34ec7bab14fc2491ca662355fddb58954fee8dd Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Thu, 19 Nov 2026 12:30:38 +0100 Subject: [PATCH 1/6] fix(standard): redact preparation records --- app/nullius/standard/computer_port.py | 6 ++++-- app/tests/test_standard_computer_port.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/app/nullius/standard/computer_port.py b/app/nullius/standard/computer_port.py index 33de6a1c..c7cb97ed 100644 --- a/app/nullius/standard/computer_port.py +++ b/app/nullius/standard/computer_port.py @@ -55,6 +55,8 @@ import time from typing import Optional, Protocol +from nullius.redact import redact + PROTOCOL = "2025-06-18" LINE_BYTES = 24 * 1024 * 1024 # one message either way, its newline counted OUT_BYTES = 2 * LINE_BYTES # answers waiting for a connection's client to read them @@ -527,8 +529,8 @@ def _prepare(self, call): does nothing), then its files read. An action the desktop did not end in time stops it.""" if self.prepared: with open(self.prepared, "a", encoding="utf-8") as f: - f.write(json.dumps({"start": time.time(), "actions": call.actions, - "files": [named for named, _ in call.files or []]}) + "\n") + f.write(json.dumps({"start": time.time(), "actions": [redact(code) for code in call.actions], + "files": [redact(named) for named, _ in call.files or []]}) + "\n") for i, code in enumerate(call.actions): if code.strip() != "WAIT" and self.desktop.python(PREPARE.format(code=code)) is None: raise Refused(f"the desktop did not end actions[{i}] in time; the actions after it were not run") diff --git a/app/tests/test_standard_computer_port.py b/app/tests/test_standard_computer_port.py index 31aec25b..e0b1f9c2 100644 --- a/app/tests/test_standard_computer_port.py +++ b/app/tests/test_standard_computer_port.py @@ -1657,6 +1657,26 @@ def setUp(self): self.bridge._begin(None) self.addCleanup(self.bridge.finish) + def test_prepare_records_redact_actions_and_file_names_without_changing_the_call(self): + secret = "sk-" + "x" * 24 + name = secret + ".txt" + actions = [f"open('agent/{name}', 'w').write({secret!r})", "WAIT"] + self.bridge.prepared = os.path.join(self.desktop.root, "prepare.jsonl") + replies = [] + call = cp.Call("prepare", {"actions": actions, "files": [name]}, + lambda c, v, i, e: replies.append((v, e))) + cp.checked(call, self.bridge.screen, self.bridge.folder) + self.bridge._serve(call) + self.assertEqual(replies, [({"files": {name: base64.b64encode(secret.encode()).decode()}}, None)]) + with open(self.bridge.prepared) as f: + records = [json.loads(line) for line in f] + self.assertEqual(len(records), 1) + record = records[0] + self.assertEqual(set(record), {"start", "actions", "files"}) + self.assertIsInstance(record["start"], float) + self.assertEqual(record["actions"], [a.replace(secret, "[REDACTED]") for a in actions]) + self.assertEqual(record["files"], ["[REDACTED].txt"]) + def submit(self, tool): args = {"actions": ["print(1)"], "files": ["/agent/a"]} if tool == "prepare" else { "path": "/agent/a", "data": "YQ=="} From 961c8ea2da3aa82cd34459b9fd168682dd8d585d Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Thu, 19 Nov 2026 12:30:49 +0100 Subject: [PATCH 2/6] docs(standard): say that the prepare record redacts credential-shaped values --- docs/ports/agent-computer-v1.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ports/agent-computer-v1.md b/docs/ports/agent-computer-v1.md index f0d8e3bd..f2f3e556 100644 --- a/docs/ports/agent-computer-v1.md +++ b/docs/ports/agent-computer-v1.md @@ -69,7 +69,7 @@ A step's `files` are read once its actions have run and their pauses have passed | Errors | An error (`isError`) is a call refused, or a call the desktop could not run or did not answer in time; it has the desktop's words. The agent may call again | | Uploads | By content: `upload` takes the bytes, in base64. A file sent once for a run: when `upload` names the file's `sha256` and the run already holds that content, the runner puts it into this unit's desktop at that call, and `data` may be left out. The run keeps a file once chunks sent from its start, in one unit, hash to the `sha256` given with the first, up to 256 MiB. Those chunks are the unit's own: they are dropped when the file starts again and at the unit's end. Nothing else passes from one unit to the next | | Connections | The server accepts every connection of the unit's session and keeps the task's steps and waiting calls outside them. A session may connect again; the task goes on when a connection closes during a step. The turn's end ends the task: Nullius sends it when the agent's process is gone or its time is up. Each connection's answers are written apart: a client that stops reading holds up no other call, and once 48 MiB of its answers wait, its connection is closed. A connection whose input has ended is closed once its calls are answered | -| The record | Every tool call goes to the unit's record as one line, `{tool, start, end, isError}`, beside the benchmark's own records; an answer over 24 MiB is an error. No argument and no result is recorded. The benchmark's records list every action's text: a step's actions pass through its loop; a `prepare` call's go to a record of their own, `{start, actions, files}` per call, written before they run | +| The record | Every tool call goes to the unit's record as one line, `{tool, start, end, isError}`, beside the benchmark's own records; an answer over 24 MiB is an error. No argument and no result is recorded. The benchmark's records list every action's text: a step's actions pass through its loop; a `prepare` call's go to a record of their own, `{start, actions, files}` per call, written before they run, with credential-shaped values replaced by `[REDACTED]` | | More tools | A benchmark may add tools, such as a question to its user. The agent finds them in `tools/list` | ## Known limits From f108a40ad657bdc50d59ffa75da4baea44addd39 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Thu, 19 Nov 2026 12:33:23 +0100 Subject: [PATCH 3/6] fix(standard): report failed preparation actions --- app/nullius/standard/computer_port.py | 13 ++++++++++--- app/tests/test_standard_computer_port.py | 20 ++++++++++++++++++++ docs/ports/agent-computer-v1.md | 2 +- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/app/nullius/standard/computer_port.py b/app/nullius/standard/computer_port.py index c7cb97ed..b76bce3b 100644 --- a/app/nullius/standard/computer_port.py +++ b/app/nullius/standard/computer_port.py @@ -164,7 +164,8 @@ def tools(screen): ("prepare", "Before the task alone, in the window between the session's start and the task: runs actions in " "the desktop, each a line of Python with pyautogui and time imported as a step's are, with no " "pause after them, then reads files, any number of paths inside folder. Returns {files}: each in " - "base64, null when there is no such file. Not a step.", + "base64, null when there is no such file. Not a step. An action that exits with an error returns " + "the desktop's words and stops the actions after it.", {"type": "object", "additionalProperties": False, "required": ["actions"], "properties": {"actions": actions, "files": FILES}}), ("click", "One step: a click at the point. Returns the screen as a PNG image after the pause.", @@ -526,14 +527,20 @@ def _file(self, path): def _prepare(self, call): """A prepare call, in the loop's thread: its record written, its actions run one at a time with no pause (WAIT - does nothing), then its files read. An action the desktop did not end in time stops it.""" + does nothing), then its files read. An action that fails or does not end in time stops it.""" if self.prepared: with open(self.prepared, "a", encoding="utf-8") as f: f.write(json.dumps({"start": time.time(), "actions": [redact(code) for code in call.actions], "files": [redact(named) for named, _ in call.files or []]}) + "\n") for i, code in enumerate(call.actions): - if code.strip() != "WAIT" and self.desktop.python(PREPARE.format(code=code)) is None: + if code.strip() == "WAIT": + continue + reply = self.desktop.python(PREPARE.format(code=code)) + if reply is None: raise Refused(f"the desktop did not end actions[{i}] in time; the actions after it were not run") + if reply.get("returncode", 0) != 0: + raise Refused(f"the desktop failed actions[{i}]: {reply.get('error') or reply.get('output') or reply}; " + "the actions after it were not run") return {"files": self._files(call.files or [])} def close_window(self): diff --git a/app/tests/test_standard_computer_port.py b/app/tests/test_standard_computer_port.py index e0b1f9c2..da3a1b5a 100644 --- a/app/tests/test_standard_computer_port.py +++ b/app/tests/test_standard_computer_port.py @@ -1686,6 +1686,26 @@ def submit(self, tool): self.bridge.submit(call) return call, replies + def test_a_failed_prepare_action_returns_its_error_and_stops_before_later_actions_and_files(self): + actions = ["WAIT", "print('before')", "raise RuntimeError('installation failed')", "print('after')"] + replies = [] + call = cp.Call("prepare", {"actions": actions, "files": ["missing"]}, + lambda c, v, i, e: replies.append((v, i, e))) + cp.checked(call, self.bridge.screen, self.bridge.folder) + self.bridge._serve(call) + value, image, error = replies[0] + self.assertIsNone(value) + self.assertIsNone(image) + self.assertIn("actions[2]", error) + self.assertIn("RuntimeError: installation failed", error) + self.assertEqual([e["code"] for e in self.desktop.events if e["kind"] == "python"], + ["folder"] + [cp.PREPARE.format(code=a) for a in actions[1:3]]) + self.assertNotIn("read", self.desktop.kinds()) + self.assertEqual(self.bridge.used, 0) + call, replies = self.submit("prepare") + self.bridge._serve(self.bridge._next()) + self.assertEqual(replies, [({"files": {"/agent/a": None}}, None, None)]) + def test_closing_refuses_queued_upload_and_prepare_as_calls_sent_after_it(self): pending = [self.submit(tool) for tool in cp.WINDOW] before = list(self.desktop.events) diff --git a/docs/ports/agent-computer-v1.md b/docs/ports/agent-computer-v1.md index f2f3e556..f7d30a4c 100644 --- a/docs/ports/agent-computer-v1.md +++ b/docs/ports/agent-computer-v1.md @@ -22,7 +22,7 @@ None of these is a step. | `info` | none | `{screen: [width, height], steps, steps_left, pause, password, folder}`. `folder`: a folder of the desktop that is the agent's own, made empty for the unit | | `screenshot` | none | The observation of the last step, as PNG. Before the first step, the picture the benchmark took at the start, before the window. It takes no new picture: two calls with no step between them return the same bytes | | `upload` | `path`, inside `folder` (absolute, or relative to it, with no `.` or `..` part); `data`, base64, at most 4 MiB a call; `append`, false (the default) to start the file and true to add to it; `sha256`, optional, the whole file's. With `sha256` and no `data`, the call takes the run's file of that `sha256` and starts the file; `append` is then refused | `{size}`: the file's size so far. In the window alone | -| `prepare` | `actions`: a list of texts, each Python run in the desktop as a step's action is, with no pause after it (`WAIT` does nothing); `files`: as for `step` | `{files: {PATH: BASE64 or null}}`, once the actions have run. In the window alone; an action the desktop does not end in time is an error and stops it | +| `prepare` | `actions`: a list of texts, each Python run in the desktop as a step's action is, with no pause after it (`WAIT` does nothing); `files`: as for `step` | `{files: {PATH: BASE64 or null}}`, once the actions have run. In the window alone; an action the desktop does not end in time is an error and stops it. An action that exits with an error returns the desktop's words and stops the actions after it. | ## Acting From 4eca6341cfbe450522c11ca211d3766a33d91a2e Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Thu, 19 Nov 2026 12:33:23 +0100 Subject: [PATCH 4/6] test(standard): make the late reference use its observation --- .../standard/kits/agent-computer-v1/driver.py | 10 ++++++---- app/tests/test_computer_kit_driver.py | 15 ++++++++++++--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/app/nullius/standard/kits/agent-computer-v1/driver.py b/app/nullius/standard/kits/agent-computer-v1/driver.py index 897d3172..286d542b 100644 --- a/app/nullius/standard/kits/agent-computer-v1/driver.py +++ b/app/nullius/standard/kits/agent-computer-v1/driver.py @@ -657,17 +657,19 @@ def late_reference(u): then the click.""" c = Client(u.socket()) try: - target = u.expected["target"] + ink = bytes.fromhex(fixture.INK[1:]) for _ in range(u.steps - 1): failed, _ = c.call("wait") failed_shot, png = c.call("screenshot") if failed or failed_shot or not isinstance(png, bytes): raise ReferenceFailed("a look by a step did not answer as a correct agent's would") - if u.up.is_set() and u.pictures[-1][0] > (u.shown or float("inf")): - break # this step's observation was taken with the button up + _, rows = fixture.png_rows(png) + point = next(((row.find(ink) // 3, y) for y, row in enumerate(rows) if row.find(ink) >= 0), None) + if point is not None: + break else: raise ReferenceFailed("the button was not seen within the steps") - failed, said = c.call("click", dict(zip("xy", centre(u.boxes[target])))) + failed, said = c.call("click", dict(zip("xy", point))) if failed: raise ReferenceFailed(f"click answered with an error: {str(said)[:200]}") finally: diff --git a/app/tests/test_computer_kit_driver.py b/app/tests/test_computer_kit_driver.py index b7b1d9b9..1e6d06ab 100644 --- a/app/tests/test_computer_kit_driver.py +++ b/app/tests/test_computer_kit_driver.py @@ -511,11 +511,12 @@ def _alive(pid): timeout=30).stdout.strip()[:1] not in ("", "Z") -def _png(text, scale, above=()): +def _png(text, scale, above=(), at=None): """A screen of 1920 by 1080 with ``text`` drawn as the fixture draws it, as PNG bytes (filter 0).""" w, h = 1920, 1080 cols, rows, cells = driver.fixture.pixels(text) - x0, y0 = (w - cols * scale) // 2, (h - rows * scale) // 2 + x, y = at or (w // 2, h // 2) + x0, y0 = x - cols * scale // 2, y - rows * scale // 2 ink = {(x0 + x * scale + dx, y0 + y * scale + dy) for x, y in cells for dx in range(scale) for dy in range(scale)} for i, label in enumerate(above): _, _, cells = driver.fixture.pixels(label) @@ -1153,7 +1154,8 @@ def start_fixture(self, values): self.log.append({"kind": "ready", "t": time.monotonic(), "boxes": dict(self.boxes)}) def capture(self): - return super().capture() + json.dumps(self.boxes).encode() + labels = self.values.get("labels", []) + return _png(" ".join(labels), 5, at=driver.centre(self.boxes[labels[0]]) if len(labels) == 1 else None) class Fixture(unittest.TestCase): @@ -1279,6 +1281,13 @@ def test_late_passes_for_the_reference_which_looks_again_by_steps(self): self.assertTrue(got["passed"], got) self.assertIn("wait", got["detail"]["calls"]) + def test_late_reference_refuses_blank_observations_even_when_capture_times_are_kept(self): + capture, blank = driver.Pictures.capture, _png("", 5) + with mock.patch.object(driver.Pictures, "capture", lambda pictures: (capture(pictures), blank)[1]): + ws, _ = self.unit("late") + with self.assertRaisesRegex(driver.ReferenceFailed, "not seen within the steps"): + driver.oracle("late", ws) + def test_late_fails_a_click_sent_with_no_look_by_a_step_once_the_button_is_up(self): ws, s = self.unit("late") u, c = driver.OPEN["late"], self.client(s) From 6875c011328c08b15343bf905714a196378e19cc Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Thu, 19 Nov 2026 12:33:23 +0100 Subject: [PATCH 5/6] docs(standard): state the late check press limit --- app/nullius/standard/kits/agent-computer-v1/standard.json | 2 +- app/tests/test_computer_kit_driver.py | 1 + docs/ports/agent-computer-v1.md | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/nullius/standard/kits/agent-computer-v1/standard.json b/app/nullius/standard/kits/agent-computer-v1/standard.json index d4bc04e8..5ab0095f 100644 --- a/app/nullius/standard/kits/agent-computer-v1/standard.json +++ b/app/nullius/standard/kits/agent-computer-v1/standard.json @@ -19,7 +19,7 @@ "scroll": "scrolls a list to read what it shows at first below its edge", "recover": "reads an acting call's error and acts again", "batch": "sends several actions in one step", - "late": "looks again with a step to find what appeared after the task was given" + "late": "looks again with a step to find what appeared after the task was given. This check does not bound the number of presses." }, "driver": { "unit_end": true, diff --git a/app/tests/test_computer_kit_driver.py b/app/tests/test_computer_kit_driver.py index 1e6d06ab..c30c1238 100644 --- a/app/tests/test_computer_kit_driver.py +++ b/app/tests/test_computer_kit_driver.py @@ -154,6 +154,7 @@ def test_the_port_names_the_kit_and_the_kit_its_checks(self): self.assertEqual(list(kit["check_details"]), list(driver.UNITS)) self.assertEqual(kit["basic"], list(checks.BASIC)) self.assertEqual(set(kit["skills"]), set(driver.UNITS) - set(checks.BASIC)) + self.assertIn("This check does not bound the number of presses.", kit["skills"]["late"]) self.assertEqual(kit["environments"]["desktop"], dict(desktop.KIT["environments"]["desktop"], summary=kit["environments"]["desktop"]["summary"])) self.assertEqual(kit["mcp_servers"]["computer"]["command"], ["python3", "{here}/pipe.py"]) # a file of its folder diff --git a/docs/ports/agent-computer-v1.md b/docs/ports/agent-computer-v1.md index f7d30a4c..c700c330 100644 --- a/docs/ports/agent-computer-v1.md +++ b/docs/ports/agent-computer-v1.md @@ -107,7 +107,7 @@ The shared code is `app/nullius/standard/computer_port.py`, with the standard li | `scroll` | skill | Scroll to the end of a list and report the large words there in order | The final message contains an uninterrupted sequence of the drawn words among at most twelve vocabulary occurrences, and the list moved | | `recover` | skill | Click a named button; the first acting call, whatever it is, is answered with an error and not run | An acting call sent after the error's answer succeeds, and the button is clicked as in `click` | | `batch` | skill | Fill three text fields with a budget of two steps | Each field holds its word | -| `late` | skill | A button appears a few seconds after the task is given, away from the screen's centre at a place drawn for the unit; click it once | The desktop logged a left press on the button with every input inside an acting call. A picture was taken at a step's end after the button appeared and before the action containing the press began. | +| `late` | skill | A button appears a few seconds after the task is given, away from the screen's centre at a place drawn for the unit; click it once | The desktop logged a left press on the button with every input inside an acting call. A picture was taken at a step's end after the button appeared and before the action containing the press began. This check does not bound the number of presses. | `see` shows three distinct common nouns drawn at 63 pixels tall, with gaps between words. `scroll` shows two at the list's end, below its initial view, at 56 pixels tall. Both sample from 96 everyday nouns chosen to avoid screen, desktop and action vocabulary and the fixture's labels. The verdict reads whole vocabulary words in the final message, ignoring case. The drawn words must occur together in the shown order at least once, with no other vocabulary word inside that sequence. Prose and punctuation may separate them. Repeated answers and vocabulary words before or after the sequence are accepted, up to twelve vocabulary occurrences in the whole message, counting repeats. `type` tests typing in a field that already has the focus; no pointing is needed. `limit` needs at least one button pressed within the two-step budget, no input after the last step and no acting call once the steps are used up. `late` draws its button 3 s after the task is given, never over the screen's centre: a click there without looking does not find it. From 5c79a6e20bb53c309db3fa0303ddc9ecef30fb92 Mon Sep 17 00:00:00 2001 From: Pierre Leca Date: Thu, 19 Nov 2026 12:33:27 +0100 Subject: [PATCH 6/6] feat(standard): record the computer kit's controls again with the new reference The kit's own proof, taken again once the reference for late finds the button in the observation it is given: the nine units' ids, the same twice and unique; no reference found in the unit's prompt (9 of 9); the null control passing 0 of 9 checks and the reference 9 of 9. Besides the two control jobs' names, the leak check's counts follow the references this run drew: 2 looked for and 7 too short, against 1 and 8 before. --- .../standard/kits/agent-computer-v1/standard.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/nullius/standard/kits/agent-computer-v1/standard.json b/app/nullius/standard/kits/agent-computer-v1/standard.json index 5ab0095f..e59d2401 100644 --- a/app/nullius/standard/kits/agent-computer-v1/standard.json +++ b/app/nullius/standard/kits/agent-computer-v1/standard.json @@ -132,10 +132,10 @@ }, "leak": { "units": 9, - "looked_for": 1, - "short": 8, + "looked_for": 2, + "short": 7, "passed": "9/9", - "note": "8 reference(s) under 12 characters not looked for: too short to tell a leak from chance", + "note": "7 reference(s) under 12 characters not looked for: too short to tell a leak from chance", "on": "2026-11-19" }, "nop": { @@ -145,7 +145,7 @@ "at_most": 0.0 }, "units": 9, - "job": "2026-11-19__10-32-52__agent-computer-v1__kit__prove-nop", + "job": "2026-11-19__13-15-24__agent-computer-v1__kit__prove-nop", "source": "nullius", "on": "2026-11-19" }, @@ -156,7 +156,7 @@ "at_least": 1.0 }, "units": 9, - "job": "2026-11-19__10-32-52__agent-computer-v1__kit__prove-oracle", + "job": "2026-11-19__13-15-24__agent-computer-v1__kit__prove-oracle", "source": "nullius", "on": "2026-11-19" }