Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions app/nullius/hosts.py
Original file line number Diff line number Diff line change
Expand Up @@ -230,8 +230,9 @@ def test(host: Host, run: Optional[Callable] = None, timeout: float = 12.0, prog
result["hostname"] = v.strip() or None
elif k == "missing" and programs and v.strip() in programs:
result["missing"].append(v.strip())
# the standard's container backend's engine there: docker when the probe finds it, else podman
result["engine"] = "docker" if result["docker"] else "podman" if result["podman"] else None
# the standard's container backend's engine there, by container.engine's rule: docker unless it does not answer and podman does
d, p = result["docker"], result["podman"]
result["engine"] = "podman" if p and (not d or (not result["docker_ok"] and result["podman_ok"])) else "docker" if d else None
result["ok"] = True
return result

Expand Down
32 changes: 27 additions & 5 deletions app/nullius/sandbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import json
import os
import re
import shutil
import subprocess
from typing import List, Optional

Expand All @@ -32,12 +33,33 @@

def docker_state() -> dict:
"""{ok, version} when the engine answers - its name and version, ``podman
6.1.4`` - else {ok: False, error}."""
6.1.4`` - else {ok: False, error}; with ``also``, the other engine when both
are installed (``_other``), so the page says why this one is in use."""
why = container.unusable()
if why:
return {"ok": False, "error": why}
r = container._docker(["version", "--format", "{{.Server.Version}}"], timeout=30, check=False)
return {"ok": True, "version": f"{container.DOCKER} {r.stdout.strip()}"}
state = {"ok": False, "error": why} if why else None
if state is None:
r = container._docker(["version", "--format", "{{.Server.Version}}"], timeout=30, check=False)
state = {"ok": True, "version": f"{container.DOCKER} {r.stdout.strip()}"}
also = _other()
return dict(state, also=also) if also else state


def _other() -> Optional[dict]:
"""The engine not in use, when it is installed too and is not the same program
(podman's ``docker``): {name, ok, said}, ``said`` its version or why it does not answer."""
other = "podman" if container.DOCKER == "docker" else "docker"
path = shutil.which(other)
if not path or (other == "docker" and container._runs_podman(path)):
return None
try:
r = subprocess.run([other, "version", "--format", "{{.Server.Version}}"], stdin=subprocess.DEVNULL,
capture_output=True, text=True, timeout=10)
except (OSError, subprocess.SubprocessError) as e:
return {"name": other, "ok": False, "said": str(e)[:200]}
if r.returncode == 0 and r.stdout.strip():
return {"name": other, "ok": True, "said": f"{other} {r.stdout.strip()}"}
lines = (r.stderr or r.stdout).strip().splitlines()
return {"name": other, "ok": False, "said": lines[-1][:200] if lines else f"exit {r.returncode}"}


def _config_dir() -> str:
Expand Down
20 changes: 17 additions & 3 deletions app/nullius/standard/container.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,11 +142,25 @@ def _runs_podman(path: str) -> bool:
return head.startswith(b"#!") and re.search(rb"^\s*exec\s+\S*podman\b", head, re.M) is not None


def _answers(program: str) -> bool:
"""Whether ``<program> ps`` succeeds, as the host probe asks (``hosts.PROBE``)."""
try:
return subprocess.run([program, "ps"], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, timeout=10).returncode == 0
except (OSError, subprocess.SubprocessError):
return False


def engine() -> str:
"""``podman`` when it is on the PATH and ``docker`` is not, or is podman
under that name; else ``docker``."""
under that name, or does not answer while podman does; else ``docker``.
Only a machine with both is asked, so one engine costs no call."""
docker, podman = shutil.which("docker"), shutil.which("podman")
return "podman" if podman and (not docker or _runs_podman(docker)) else "docker"
if not podman:
return "docker"
if not docker or _runs_podman(docker):
return "podman"
return "podman" if not _answers(docker) and _answers(podman) else "docker"


DOCKER = engine() # the engine's program, chosen once
Expand Down Expand Up @@ -271,7 +285,7 @@ def unusable() -> Optional[str]:
hint = (" (add this user to the docker group, then log in again)"
if not podman() and "permission denied" in r.stderr.lower() else "")
if not podman() and shutil.which("podman"): # docker wins (engine), so say podman is there
hint += ". podman is on this machine: Nullius uses it when docker is not on the PATH"
hint += ". podman is on this machine: Nullius uses it when docker does not answer and podman does"
return f"{DOCKER} is installed but not usable by this user: " + (said[-1][:200] if said else f"exit {r.returncode}") + hint
if podman():
try:
Expand Down
3 changes: 3 additions & 0 deletions app/static/settings/sandbox.js
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@
const docker = $('[data-docker-state]');
docker.textContent = j.docker.ok ? '✓ ' + j.docker.version : '✕ ' + j.docker.error;
docker.className = 'st-row-value ' + (j.docker.ok ? 'ok' : 'bad');
const also = j.docker.also, other = $('[data-docker-also]');
other.hidden = !also;
if (also) other.lastElementChild.textContent = also.ok ? also.said + ' is installed too' : also.name + ' is installed and does not answer: ' + also.said;
boat = j.boat || {set: false, env: false};
render();
};
Expand Down
2 changes: 1 addition & 1 deletion app/templates/settings/settings.html
Original file line number Diff line number Diff line change
Expand Up @@ -395,7 +395,7 @@ <h2 id="judge-h">Judges</h2>
<section class="st-section" id="sandbox" aria-labelledby="sandbox-h" data-sandbox="{{ url_for('settings_sandbox') }}"{% if writable %} data-csrf="{{ csrf }}" data-boat="{{ url_for('settings_sandbox_boat') }}"{% endif %}>
<div class="st-section-head"><h2 id="sandbox-h">Sandbox</h2><p>Where agents run, apart from this machine.</p></div>
<div class="st-card">
<div class="st-row"><span class="st-row-label">Engine</span><span class="st-row-value" data-docker-state>Checking…</span></div>
<div class="st-row"><span class="st-row-label">Engine</span><span class="st-row-value" data-docker-state>Checking…</span></div><div class="st-row" data-docker-also hidden><span class="st-row-label"></span><span class="st-row-value faint"></span></div>
</div>
<div class="st-card" data-boat-card>
<div class="st-provider-summary"><img class="st-logo on-light-bg" src="{{ url_for('static', filename='brand/boat.svg') }}" alt="" width="28" height="28"><img class="st-logo on-dark-bg" src="{{ url_for('static', filename='brand/boat-dark.svg') }}" alt="" width="28" height="28"><div><h3>Boat</h3><p class="st-about" data-boat-state>Checking…</p></div>{% if writable %}<button type="button" class="btn sm ghost" data-boat-edit>Edit</button><button type="button" class="btn sm ghost danger" data-boat-remove aria-expanded="false" aria-controls="boat-remove-ask" hidden>Remove</button>{% endif %}</div>
Expand Down
6 changes: 6 additions & 0 deletions app/tests/test_hosts.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,12 @@ def test_missing_tools_are_said_not_failed(self):
r = hosts.test(hosts.parse("bench"), run=lambda a, t: (0, "ok\ndocker=no\npodman=yes\npodman_ok=yes\n", ""))
self.assertEqual(r["engine"], "podman")

def test_the_engine_is_the_one_that_answers(self):
for said, want in (("docker_ok=no\npodman_ok=yes", "podman"), ("docker_ok=yes\npodman_ok=yes", "docker"),
("docker_ok=no\npodman_ok=no", "docker")): # neither: docker, whose reason is said
r = hosts.test(hosts.parse("bench"), run=lambda a, t: (0, f"ok\ndocker=yes\npodman=yes\n{said}\n", ""))
self.assertEqual(r["engine"], want, said)

def test_the_probe_asks_each_engine_that_is_there(self):
with tempfile.TemporaryDirectory() as d:
for name, code in (("podman", 0), ("uv", 0)):
Expand Down
3 changes: 2 additions & 1 deletion app/tests/test_sandbox.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ def test_podman_logs_in_to_docker_hub_by_name_and_says_who(self):

def test_the_engine_is_named_with_its_version(self):
with mock.patch.object(container, "DOCKER", "podman"), mock.patch.object(container, "unusable", return_value=None), \
mock.patch.object(container, "_docker", return_value=done(out="6.1.4\n")):
mock.patch.object(container, "_docker", return_value=done(out="6.1.4\n")), \
mock.patch.object(sandbox.shutil, "which", return_value=None): # no other engine: the machine's own is not asked
self.assertEqual(sandbox.docker_state(), {"ok": True, "version": "podman 6.1.4"})

def test_a_bad_username_or_empty_token_asks_docker_nothing(self):
Expand Down
37 changes: 36 additions & 1 deletion app/tests/test_standard_container.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,41 @@ def test_docker_is_the_engine_unless_podman_stands_in_for_it(self):
os.symlink(podman, docker)
self.assertEqual(self.engine(), "podman")

def test_with_both_on_the_path_the_one_that_answers_is_the_engine(self):
refuses = "#!/bin/sh\necho 'permission denied while trying to connect to the Docker daemon socket' >&2\nexit 1\n"
for docker, podman, want in ((refuses, "#!/bin/sh\nexit 0\n", "podman"), # docker refuses this user, podman answers
("#!/bin/sh\nexit 0\n", "#!/bin/sh\nexit 0\n", "docker"),
(refuses, "#!/bin/sh\nexit 125\n", "docker")): # neither: docker, so its reason is said
self.program("docker", docker); self.program("podman", podman)
with self.subTest(want=want):
self.assertEqual(self.engine(), want)

def test_the_sandbox_says_the_other_engine_when_both_are_installed(self):
from nullius import sandbox
def other(engine):
with mock.patch.dict(os.environ, {"PATH": self.tmp}), mock.patch.object(container, "DOCKER", engine):
return sandbox._other()
self.program("podman", "#!/bin/sh\necho 5.4.2\n")
self.assertIsNone(other("podman")) # podman alone: nothing else to say
self.program("docker", "#!/bin/sh\necho 'Cannot connect to the Docker daemon at unix:///var/run/docker.sock' >&2\nexit 1\n")
self.assertEqual(other("podman"), {"name": "docker", "ok": False,
"said": "Cannot connect to the Docker daemon at unix:///var/run/docker.sock"})
self.assertEqual(other("docker"), {"name": "podman", "ok": True, "said": "podman 5.4.2"})
self.program("docker", "#!/bin/sh\nexec podman \"$@\"\n") # podman-docker's docker: the same program
self.assertIsNone(other("podman"))

def test_the_host_probe_and_the_run_there_pick_the_same_engine(self):
from nullius import hosts
for tool in ("sh", "sed", "grep", "hostname"):
os.symlink(shutil.which(tool), os.path.join(self.tmp, tool))
self.program("docker", "#!/bin/sh\necho 'permission denied' >&2\nexit 1\n")
self.program("podman", "#!/bin/sh\nexit 0\n")
run = lambda argv, timeout: (lambda p: (p.returncode, p.stdout, p.stderr))(
subprocess.run(["sh", "-c", argv[-1]], capture_output=True, text=True, env={"PATH": self.tmp}, timeout=timeout))
got = hosts.test(hosts.parse("bench"), run=run)
self.assertEqual((got["docker_ok"], got["podman_ok"], got["engine"]), (False, True, "podman"))
self.assertEqual(self.engine(), got["engine"])

def test_a_docker_that_refuses_this_user_says_podman_is_there(self):
self.enterContext(mock.patch.object(container, "DOCKER", "docker"))
refused = subprocess.CompletedProcess([], 1, "", "permission denied while trying to connect to the docker API\n")
Expand All @@ -84,7 +119,7 @@ def test_a_docker_that_refuses_this_user_says_podman_is_there(self):
with mock.patch.object(container.shutil, "which", return_value=None):
self.assertEqual(container.unusable(), said)
with mock.patch.object(container.shutil, "which", return_value="/usr/bin/podman"):
self.assertEqual(container.unusable(), said + ". podman is on this machine: Nullius uses it when docker is not on the PATH")
self.assertEqual(container.unusable(), said + ". podman is on this machine: Nullius uses it when docker does not answer and podman does")

def test_podman_is_asked_whether_it_is_rootless_in_its_own_terms(self):
self.enterContext(mock.patch.object(container, "DOCKER", "podman"))
Expand Down
2 changes: 1 addition & 1 deletion docs/standard.md
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ record); verifier/reward.txt and score.json
| Units | Run `-j` at a time, each with its own workspace (a git repository of its own), home and relay |
| A relay per unit | Its record is the trial's `.work/relay`; the judge and simulated user are routes on it (`<unit>-judge`, `<unit>-simulator`). An option whose driver embeds gets a relay of the run's own before any unit, recorded in `.work/relay-benchmark` as the benchmark's cost. A 200 with no reply, not streamed, reaches the caller as a 502 (or the 429 or 402 its error held); the record keeps `answered: 200`. A call refused with 400, 413 or 422 records its inline images' types and sizes as `images`, never their bytes |
| Job name | One run at a time per job name: another is refused before it writes anything. A later run under the same name moves each trial's earlier record aside to the trial's `.work/earlier.<ns>` |
| Backends (`--backend`) | `local`: processes on this machine. `container`: each unit's agent or capability in a container, its relay beside it (`standard/container.py`), on Docker, or on Podman 4.7 or later where `docker` is missing or is podman under that name. Podman needs no Docker API socket and no `containers.conf` change. Where a `docker` program is on the PATH, Nullius runs it even when it does not answer. The reason it gives then says that podman is on the machine. `docker`, the old name of `container`, is still accepted. For agent and capability runs the driver stays on the machine; Original setups instead containerize the driver and its answerer (`standard/setup_container.py`); in a container a check that runs what the agent left (the workspace kit's test) runs it in a throwaway, network-less container of the unit's image (`container.check_argv`), and no check reads through a link the agent planted; `config.json` records the backend and the image. `boat`: each unit's agent or capability in a boat.dev VM of its own (`standard/boat.py`, [The boat backend](#the-boat-backend)) |
| Backends (`--backend`) | `local`: processes on this machine. `container`: each unit's agent or capability in a container, its relay beside it (`standard/container.py`), on Docker, or on Podman 4.7 or later where `docker` is missing, is podman under that name, or does not answer (`docker ps` fails) while `podman ps` succeeds. Podman needs no Docker API socket and no `containers.conf` change. Where neither answers, Nullius keeps `docker` and its reason says that podman is on the machine. A host is probed by the same rule. `docker`, the old name of `container`, is still accepted. For agent and capability runs the driver stays on the machine; Original setups instead containerize the driver and its answerer (`standard/setup_container.py`); in a container a check that runs what the agent left (the workspace kit's test) runs it in a throwaway, network-less container of the unit's image (`container.check_argv`), and no check reads through a link the agent planted; `config.json` records the backend and the image. `boat`: each unit's agent or capability in a boat.dev VM of its own (`standard/boat.py`, [The boat backend](#the-boat-backend)) |
| Progress | One line per stage on stderr and in `progress.jsonl`: `[nullius] stage=install agent=codex@2.2.2 state=done seconds=41 at=...` (`start`, `done`, `cached`, `failed`; a download's bytes every 5 s) for the base image, an environment, the upstream, a dataset, `driver.install` and the agent; then `stage=units n=10`, `stage=unit i=3 of=10 unit=... outcome=passed`, `stage=done`; `stage=refused` alone for a run refused first. From `stage=units` the job also writes `events.jsonl`, Harbor's live feed |
| Outcomes | A last model call that got a 429 is `rate_limited` (calls in the order they began; a side model's later 200 does not cover the agent's own), a refused key an API error, both outside the agent's rate, unless the unit passed; a unit asked to stop (a `stop` beside its relay's records) keeps the score its driver gave despite the agent's own last 429; the agent's own last call over the model's context window is `ContextWindowExceededError`, the agent's; an agent past its limit for the unit, every turn included, is the agent's timeout ([time limits](cli/run.md#time-limits)). `"kind": "score"` leaves a score and no pass |
| Rate limits | After a `rate_limited` end no unit starts for 60 s, or the provider's `Retry-After`, doubling to 300 s while units keep ending so |
Expand Down
Loading