diff --git a/app/nullius/standard/computer_port.py b/app/nullius/standard/computer_port.py index 33de6a1c..b76bce3b 100644 --- a/app/nullius/standard/computer_port.py +++ b/app/nullius/standard/computer_port.py @@ -55,6 +55,8 @@ import time from typing import Optional, Protocol +from nullius.redact import redact + PROTOCOL = "2025-06-18" LINE_BYTES = 24 * 1024 * 1024 # one message either way, its newline counted OUT_BYTES = 2 * LINE_BYTES # answers waiting for a connection's client to read them @@ -162,7 +164,8 @@ def tools(screen): ("prepare", "Before the task alone, in the window between the session's start and the task: runs actions in " "the desktop, each a line of Python with pyautogui and time imported as a step's are, with no " "pause after them, then reads files, any number of paths inside folder. Returns {files}: each in " - "base64, null when there is no such file. Not a step.", + "base64, null when there is no such file. Not a step. An action that exits with an error returns " + "the desktop's words and stops the actions after it.", {"type": "object", "additionalProperties": False, "required": ["actions"], "properties": {"actions": actions, "files": FILES}}), ("click", "One step: a click at the point. Returns the screen as a PNG image after the pause.", @@ -524,14 +527,20 @@ def _file(self, path): def _prepare(self, call): """A prepare call, in the loop's thread: its record written, its actions run one at a time with no pause (WAIT - does nothing), then its files read. An action the desktop did not end in time stops it.""" + does nothing), then its files read. An action that fails or does not end in time stops it.""" if self.prepared: with open(self.prepared, "a", encoding="utf-8") as f: - f.write(json.dumps({"start": time.time(), "actions": call.actions, - "files": [named for named, _ in call.files or []]}) + "\n") + f.write(json.dumps({"start": time.time(), "actions": [redact(code) for code in call.actions], + "files": [redact(named) for named, _ in call.files or []]}) + "\n") for i, code in enumerate(call.actions): - if code.strip() != "WAIT" and self.desktop.python(PREPARE.format(code=code)) is None: + if code.strip() == "WAIT": + continue + reply = self.desktop.python(PREPARE.format(code=code)) + if reply is None: raise Refused(f"the desktop did not end actions[{i}] in time; the actions after it were not run") + if reply.get("returncode", 0) != 0: + raise Refused(f"the desktop failed actions[{i}]: {reply.get('error') or reply.get('output') or reply}; " + "the actions after it were not run") return {"files": self._files(call.files or [])} def close_window(self): diff --git a/app/nullius/standard/kits/agent-computer-v1/driver.py b/app/nullius/standard/kits/agent-computer-v1/driver.py index 897d3172..286d542b 100644 --- a/app/nullius/standard/kits/agent-computer-v1/driver.py +++ b/app/nullius/standard/kits/agent-computer-v1/driver.py @@ -657,17 +657,19 @@ def late_reference(u): then the click.""" c = Client(u.socket()) try: - target = u.expected["target"] + ink = bytes.fromhex(fixture.INK[1:]) for _ in range(u.steps - 1): failed, _ = c.call("wait") failed_shot, png = c.call("screenshot") if failed or failed_shot or not isinstance(png, bytes): raise ReferenceFailed("a look by a step did not answer as a correct agent's would") - if u.up.is_set() and u.pictures[-1][0] > (u.shown or float("inf")): - break # this step's observation was taken with the button up + _, rows = fixture.png_rows(png) + point = next(((row.find(ink) // 3, y) for y, row in enumerate(rows) if row.find(ink) >= 0), None) + if point is not None: + break else: raise ReferenceFailed("the button was not seen within the steps") - failed, said = c.call("click", dict(zip("xy", centre(u.boxes[target])))) + failed, said = c.call("click", dict(zip("xy", point))) if failed: raise ReferenceFailed(f"click answered with an error: {str(said)[:200]}") finally: diff --git a/app/nullius/standard/kits/agent-computer-v1/standard.json b/app/nullius/standard/kits/agent-computer-v1/standard.json index d4bc04e8..e59d2401 100644 --- a/app/nullius/standard/kits/agent-computer-v1/standard.json +++ b/app/nullius/standard/kits/agent-computer-v1/standard.json @@ -19,7 +19,7 @@ "scroll": "scrolls a list to read what it shows at first below its edge", "recover": "reads an acting call's error and acts again", "batch": "sends several actions in one step", - "late": "looks again with a step to find what appeared after the task was given" + "late": "looks again with a step to find what appeared after the task was given. This check does not bound the number of presses." }, "driver": { "unit_end": true, @@ -132,10 +132,10 @@ }, "leak": { "units": 9, - "looked_for": 1, - "short": 8, + "looked_for": 2, + "short": 7, "passed": "9/9", - "note": "8 reference(s) under 12 characters not looked for: too short to tell a leak from chance", + "note": "7 reference(s) under 12 characters not looked for: too short to tell a leak from chance", "on": "2026-11-19" }, "nop": { @@ -145,7 +145,7 @@ "at_most": 0.0 }, "units": 9, - "job": "2026-11-19__10-32-52__agent-computer-v1__kit__prove-nop", + "job": "2026-11-19__13-15-24__agent-computer-v1__kit__prove-nop", "source": "nullius", "on": "2026-11-19" }, @@ -156,7 +156,7 @@ "at_least": 1.0 }, "units": 9, - "job": "2026-11-19__10-32-52__agent-computer-v1__kit__prove-oracle", + "job": "2026-11-19__13-15-24__agent-computer-v1__kit__prove-oracle", "source": "nullius", "on": "2026-11-19" } diff --git a/app/tests/test_computer_kit_driver.py b/app/tests/test_computer_kit_driver.py index b7b1d9b9..c30c1238 100644 --- a/app/tests/test_computer_kit_driver.py +++ b/app/tests/test_computer_kit_driver.py @@ -154,6 +154,7 @@ def test_the_port_names_the_kit_and_the_kit_its_checks(self): self.assertEqual(list(kit["check_details"]), list(driver.UNITS)) self.assertEqual(kit["basic"], list(checks.BASIC)) self.assertEqual(set(kit["skills"]), set(driver.UNITS) - set(checks.BASIC)) + self.assertIn("This check does not bound the number of presses.", kit["skills"]["late"]) self.assertEqual(kit["environments"]["desktop"], dict(desktop.KIT["environments"]["desktop"], summary=kit["environments"]["desktop"]["summary"])) self.assertEqual(kit["mcp_servers"]["computer"]["command"], ["python3", "{here}/pipe.py"]) # a file of its folder @@ -511,11 +512,12 @@ def _alive(pid): timeout=30).stdout.strip()[:1] not in ("", "Z") -def _png(text, scale, above=()): +def _png(text, scale, above=(), at=None): """A screen of 1920 by 1080 with ``text`` drawn as the fixture draws it, as PNG bytes (filter 0).""" w, h = 1920, 1080 cols, rows, cells = driver.fixture.pixels(text) - x0, y0 = (w - cols * scale) // 2, (h - rows * scale) // 2 + x, y = at or (w // 2, h // 2) + x0, y0 = x - cols * scale // 2, y - rows * scale // 2 ink = {(x0 + x * scale + dx, y0 + y * scale + dy) for x, y in cells for dx in range(scale) for dy in range(scale)} for i, label in enumerate(above): _, _, cells = driver.fixture.pixels(label) @@ -1153,7 +1155,8 @@ def start_fixture(self, values): self.log.append({"kind": "ready", "t": time.monotonic(), "boxes": dict(self.boxes)}) def capture(self): - return super().capture() + json.dumps(self.boxes).encode() + labels = self.values.get("labels", []) + return _png(" ".join(labels), 5, at=driver.centre(self.boxes[labels[0]]) if len(labels) == 1 else None) class Fixture(unittest.TestCase): @@ -1279,6 +1282,13 @@ def test_late_passes_for_the_reference_which_looks_again_by_steps(self): self.assertTrue(got["passed"], got) self.assertIn("wait", got["detail"]["calls"]) + def test_late_reference_refuses_blank_observations_even_when_capture_times_are_kept(self): + capture, blank = driver.Pictures.capture, _png("", 5) + with mock.patch.object(driver.Pictures, "capture", lambda pictures: (capture(pictures), blank)[1]): + ws, _ = self.unit("late") + with self.assertRaisesRegex(driver.ReferenceFailed, "not seen within the steps"): + driver.oracle("late", ws) + def test_late_fails_a_click_sent_with_no_look_by_a_step_once_the_button_is_up(self): ws, s = self.unit("late") u, c = driver.OPEN["late"], self.client(s) diff --git a/app/tests/test_standard_computer_port.py b/app/tests/test_standard_computer_port.py index 31aec25b..da3a1b5a 100644 --- a/app/tests/test_standard_computer_port.py +++ b/app/tests/test_standard_computer_port.py @@ -1657,6 +1657,26 @@ def setUp(self): self.bridge._begin(None) self.addCleanup(self.bridge.finish) + def test_prepare_records_redact_actions_and_file_names_without_changing_the_call(self): + secret = "sk-" + "x" * 24 + name = secret + ".txt" + actions = [f"open('agent/{name}', 'w').write({secret!r})", "WAIT"] + self.bridge.prepared = os.path.join(self.desktop.root, "prepare.jsonl") + replies = [] + call = cp.Call("prepare", {"actions": actions, "files": [name]}, + lambda c, v, i, e: replies.append((v, e))) + cp.checked(call, self.bridge.screen, self.bridge.folder) + self.bridge._serve(call) + self.assertEqual(replies, [({"files": {name: base64.b64encode(secret.encode()).decode()}}, None)]) + with open(self.bridge.prepared) as f: + records = [json.loads(line) for line in f] + self.assertEqual(len(records), 1) + record = records[0] + self.assertEqual(set(record), {"start", "actions", "files"}) + self.assertIsInstance(record["start"], float) + self.assertEqual(record["actions"], [a.replace(secret, "[REDACTED]") for a in actions]) + self.assertEqual(record["files"], ["[REDACTED].txt"]) + def submit(self, tool): args = {"actions": ["print(1)"], "files": ["/agent/a"]} if tool == "prepare" else { "path": "/agent/a", "data": "YQ=="} @@ -1666,6 +1686,26 @@ def submit(self, tool): self.bridge.submit(call) return call, replies + def test_a_failed_prepare_action_returns_its_error_and_stops_before_later_actions_and_files(self): + actions = ["WAIT", "print('before')", "raise RuntimeError('installation failed')", "print('after')"] + replies = [] + call = cp.Call("prepare", {"actions": actions, "files": ["missing"]}, + lambda c, v, i, e: replies.append((v, i, e))) + cp.checked(call, self.bridge.screen, self.bridge.folder) + self.bridge._serve(call) + value, image, error = replies[0] + self.assertIsNone(value) + self.assertIsNone(image) + self.assertIn("actions[2]", error) + self.assertIn("RuntimeError: installation failed", error) + self.assertEqual([e["code"] for e in self.desktop.events if e["kind"] == "python"], + ["folder"] + [cp.PREPARE.format(code=a) for a in actions[1:3]]) + self.assertNotIn("read", self.desktop.kinds()) + self.assertEqual(self.bridge.used, 0) + call, replies = self.submit("prepare") + self.bridge._serve(self.bridge._next()) + self.assertEqual(replies, [({"files": {"/agent/a": None}}, None, None)]) + def test_closing_refuses_queued_upload_and_prepare_as_calls_sent_after_it(self): pending = [self.submit(tool) for tool in cp.WINDOW] before = list(self.desktop.events) diff --git a/docs/ports/agent-computer-v1.md b/docs/ports/agent-computer-v1.md index f0d8e3bd..c700c330 100644 --- a/docs/ports/agent-computer-v1.md +++ b/docs/ports/agent-computer-v1.md @@ -22,7 +22,7 @@ None of these is a step. | `info` | none | `{screen: [width, height], steps, steps_left, pause, password, folder}`. `folder`: a folder of the desktop that is the agent's own, made empty for the unit | | `screenshot` | none | The observation of the last step, as PNG. Before the first step, the picture the benchmark took at the start, before the window. It takes no new picture: two calls with no step between them return the same bytes | | `upload` | `path`, inside `folder` (absolute, or relative to it, with no `.` or `..` part); `data`, base64, at most 4 MiB a call; `append`, false (the default) to start the file and true to add to it; `sha256`, optional, the whole file's. With `sha256` and no `data`, the call takes the run's file of that `sha256` and starts the file; `append` is then refused | `{size}`: the file's size so far. In the window alone | -| `prepare` | `actions`: a list of texts, each Python run in the desktop as a step's action is, with no pause after it (`WAIT` does nothing); `files`: as for `step` | `{files: {PATH: BASE64 or null}}`, once the actions have run. In the window alone; an action the desktop does not end in time is an error and stops it | +| `prepare` | `actions`: a list of texts, each Python run in the desktop as a step's action is, with no pause after it (`WAIT` does nothing); `files`: as for `step` | `{files: {PATH: BASE64 or null}}`, once the actions have run. In the window alone; an action the desktop does not end in time is an error and stops it. An action that exits with an error returns the desktop's words and stops the actions after it. | ## Acting @@ -69,7 +69,7 @@ A step's `files` are read once its actions have run and their pauses have passed | Errors | An error (`isError`) is a call refused, or a call the desktop could not run or did not answer in time; it has the desktop's words. The agent may call again | | Uploads | By content: `upload` takes the bytes, in base64. A file sent once for a run: when `upload` names the file's `sha256` and the run already holds that content, the runner puts it into this unit's desktop at that call, and `data` may be left out. The run keeps a file once chunks sent from its start, in one unit, hash to the `sha256` given with the first, up to 256 MiB. Those chunks are the unit's own: they are dropped when the file starts again and at the unit's end. Nothing else passes from one unit to the next | | Connections | The server accepts every connection of the unit's session and keeps the task's steps and waiting calls outside them. A session may connect again; the task goes on when a connection closes during a step. The turn's end ends the task: Nullius sends it when the agent's process is gone or its time is up. Each connection's answers are written apart: a client that stops reading holds up no other call, and once 48 MiB of its answers wait, its connection is closed. A connection whose input has ended is closed once its calls are answered | -| The record | Every tool call goes to the unit's record as one line, `{tool, start, end, isError}`, beside the benchmark's own records; an answer over 24 MiB is an error. No argument and no result is recorded. The benchmark's records list every action's text: a step's actions pass through its loop; a `prepare` call's go to a record of their own, `{start, actions, files}` per call, written before they run | +| The record | Every tool call goes to the unit's record as one line, `{tool, start, end, isError}`, beside the benchmark's own records; an answer over 24 MiB is an error. No argument and no result is recorded. The benchmark's records list every action's text: a step's actions pass through its loop; a `prepare` call's go to a record of their own, `{start, actions, files}` per call, written before they run, with credential-shaped values replaced by `[REDACTED]` | | More tools | A benchmark may add tools, such as a question to its user. The agent finds them in `tools/list` | ## Known limits @@ -107,7 +107,7 @@ The shared code is `app/nullius/standard/computer_port.py`, with the standard li | `scroll` | skill | Scroll to the end of a list and report the large words there in order | The final message contains an uninterrupted sequence of the drawn words among at most twelve vocabulary occurrences, and the list moved | | `recover` | skill | Click a named button; the first acting call, whatever it is, is answered with an error and not run | An acting call sent after the error's answer succeeds, and the button is clicked as in `click` | | `batch` | skill | Fill three text fields with a budget of two steps | Each field holds its word | -| `late` | skill | A button appears a few seconds after the task is given, away from the screen's centre at a place drawn for the unit; click it once | The desktop logged a left press on the button with every input inside an acting call. A picture was taken at a step's end after the button appeared and before the action containing the press began. | +| `late` | skill | A button appears a few seconds after the task is given, away from the screen's centre at a place drawn for the unit; click it once | The desktop logged a left press on the button with every input inside an acting call. A picture was taken at a step's end after the button appeared and before the action containing the press began. This check does not bound the number of presses. | `see` shows three distinct common nouns drawn at 63 pixels tall, with gaps between words. `scroll` shows two at the list's end, below its initial view, at 56 pixels tall. Both sample from 96 everyday nouns chosen to avoid screen, desktop and action vocabulary and the fixture's labels. The verdict reads whole vocabulary words in the final message, ignoring case. The drawn words must occur together in the shown order at least once, with no other vocabulary word inside that sequence. Prose and punctuation may separate them. Repeated answers and vocabulary words before or after the sequence are accepted, up to twelve vocabulary occurrences in the whole message, counting repeats. `type` tests typing in a field that already has the focus; no pointing is needed. `limit` needs at least one button pressed within the two-step budget, no input after the last step and no acting call once the steps are used up. `late` draws its button 3 s after the task is given, never over the screen's centre: a click there without looking does not find it.