From 09170e33e696d9d8cd3f83f7aedfa1f317362730 Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Thu, 19 Nov 2026 13:50:14 +0100 Subject: [PATCH 1/3] fix(standard): use podman when docker is installed but does not answer With both docker and podman on the PATH, the container backend chose docker by presence alone, so a docker that refuses this user or has no daemon blocked the standard's container runs even where podman works. The engine is now docker unless `docker ps` fails and `podman ps` succeeds, on this machine (container.engine) and in a host's probe (hosts.test) alike, so the probe and the standard's run there agree. Only a machine with both is asked; with one engine the choice costs no call. Where neither answers, docker is kept and its reason is said as before. --- app/nullius/hosts.py | 5 +++-- app/nullius/standard/container.py | 20 +++++++++++++++++--- app/tests/test_hosts.py | 6 ++++++ app/tests/test_standard_container.py | 23 ++++++++++++++++++++++- docs/standard.md | 2 +- 5 files changed, 49 insertions(+), 7 deletions(-) diff --git a/app/nullius/hosts.py b/app/nullius/hosts.py index 3a4ebdea..277704e1 100644 --- a/app/nullius/hosts.py +++ b/app/nullius/hosts.py @@ -230,8 +230,9 @@ def test(host: Host, run: Optional[Callable] = None, timeout: float = 12.0, prog result["hostname"] = v.strip() or None elif k == "missing" and programs and v.strip() in programs: result["missing"].append(v.strip()) - # the standard's container backend's engine there: docker when the probe finds it, else podman - result["engine"] = "docker" if result["docker"] else "podman" if result["podman"] else None + # the standard's container backend's engine there, by container.engine's rule: docker unless it does not answer and podman does + d, p = result["docker"], result["podman"] + result["engine"] = "podman" if p and (not d or (not result["docker_ok"] and result["podman_ok"])) else "docker" if d else None result["ok"] = True return result diff --git a/app/nullius/standard/container.py b/app/nullius/standard/container.py index 88a0941e..94e1a04a 100644 --- a/app/nullius/standard/container.py +++ b/app/nullius/standard/container.py @@ -142,11 +142,25 @@ def _runs_podman(path: str) -> bool: return head.startswith(b"#!") and re.search(rb"^\s*exec\s+\S*podman\b", head, re.M) is not None +def _answers(program: str) -> bool: + """Whether `` ps`` succeeds, as the host probe asks (``hosts.PROBE``).""" + try: + return subprocess.run([program, "ps"], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=10).returncode == 0 + except (OSError, subprocess.SubprocessError): + return False + + def engine() -> str: """``podman`` when it is on the PATH and ``docker`` is not, or is podman - under that name; else ``docker``.""" + under that name, or does not answer while podman does; else ``docker``. + Only a machine with both is asked, so one engine costs no call.""" docker, podman = shutil.which("docker"), shutil.which("podman") - return "podman" if podman and (not docker or _runs_podman(docker)) else "docker" + if not podman: + return "docker" + if not docker or _runs_podman(docker): + return "podman" + return "podman" if not _answers(docker) and _answers(podman) else "docker" DOCKER = engine() # the engine's program, chosen once @@ -271,7 +285,7 @@ def unusable() -> Optional[str]: hint = (" (add this user to the docker group, then log in again)" if not podman() and "permission denied" in r.stderr.lower() else "") if not podman() and shutil.which("podman"): # docker wins (engine), so say podman is there - hint += ". podman is on this machine: Nullius uses it when docker is not on the PATH" + hint += ". podman is on this machine: Nullius uses it when docker does not answer and podman does" return f"{DOCKER} is installed but not usable by this user: " + (said[-1][:200] if said else f"exit {r.returncode}") + hint if podman(): try: diff --git a/app/tests/test_hosts.py b/app/tests/test_hosts.py index 79fcf1c5..64801dbf 100644 --- a/app/tests/test_hosts.py +++ b/app/tests/test_hosts.py @@ -114,6 +114,12 @@ def test_missing_tools_are_said_not_failed(self): r = hosts.test(hosts.parse("bench"), run=lambda a, t: (0, "ok\ndocker=no\npodman=yes\npodman_ok=yes\n", "")) self.assertEqual(r["engine"], "podman") + def test_the_engine_is_the_one_that_answers(self): + for said, want in (("docker_ok=no\npodman_ok=yes", "podman"), ("docker_ok=yes\npodman_ok=yes", "docker"), + ("docker_ok=no\npodman_ok=no", "docker")): # neither: docker, whose reason is said + r = hosts.test(hosts.parse("bench"), run=lambda a, t: (0, f"ok\ndocker=yes\npodman=yes\n{said}\n", "")) + self.assertEqual(r["engine"], want, said) + def test_the_probe_asks_each_engine_that_is_there(self): with tempfile.TemporaryDirectory() as d: for name, code in (("podman", 0), ("uv", 0)): diff --git a/app/tests/test_standard_container.py b/app/tests/test_standard_container.py index c4c2f4a5..bd6628a6 100644 --- a/app/tests/test_standard_container.py +++ b/app/tests/test_standard_container.py @@ -75,6 +75,27 @@ def test_docker_is_the_engine_unless_podman_stands_in_for_it(self): os.symlink(podman, docker) self.assertEqual(self.engine(), "podman") + def test_with_both_on_the_path_the_one_that_answers_is_the_engine(self): + refuses = "#!/bin/sh\necho 'permission denied while trying to connect to the Docker daemon socket' >&2\nexit 1\n" + for docker, podman, want in ((refuses, "#!/bin/sh\nexit 0\n", "podman"), # docker refuses this user, podman answers + ("#!/bin/sh\nexit 0\n", "#!/bin/sh\nexit 0\n", "docker"), + (refuses, "#!/bin/sh\nexit 125\n", "docker")): # neither: docker, so its reason is said + self.program("docker", docker); self.program("podman", podman) + with self.subTest(want=want): + self.assertEqual(self.engine(), want) + + def test_the_host_probe_and_the_run_there_pick_the_same_engine(self): + from nullius import hosts + for tool in ("sh", "sed", "grep", "hostname"): + os.symlink(shutil.which(tool), os.path.join(self.tmp, tool)) + self.program("docker", "#!/bin/sh\necho 'permission denied' >&2\nexit 1\n") + self.program("podman", "#!/bin/sh\nexit 0\n") + run = lambda argv, timeout: (lambda p: (p.returncode, p.stdout, p.stderr))( + subprocess.run(["sh", "-c", argv[-1]], capture_output=True, text=True, env={"PATH": self.tmp}, timeout=timeout)) + got = hosts.test(hosts.parse("bench"), run=run) + self.assertEqual((got["docker_ok"], got["podman_ok"], got["engine"]), (False, True, "podman")) + self.assertEqual(self.engine(), got["engine"]) + def test_a_docker_that_refuses_this_user_says_podman_is_there(self): self.enterContext(mock.patch.object(container, "DOCKER", "docker")) refused = subprocess.CompletedProcess([], 1, "", "permission denied while trying to connect to the docker API\n") @@ -84,7 +105,7 @@ def test_a_docker_that_refuses_this_user_says_podman_is_there(self): with mock.patch.object(container.shutil, "which", return_value=None): self.assertEqual(container.unusable(), said) with mock.patch.object(container.shutil, "which", return_value="/usr/bin/podman"): - self.assertEqual(container.unusable(), said + ". podman is on this machine: Nullius uses it when docker is not on the PATH") + self.assertEqual(container.unusable(), said + ". podman is on this machine: Nullius uses it when docker does not answer and podman does") def test_podman_is_asked_whether_it_is_rootless_in_its_own_terms(self): self.enterContext(mock.patch.object(container, "DOCKER", "podman")) diff --git a/docs/standard.md b/docs/standard.md index c888368b..ecc1221d 100644 --- a/docs/standard.md +++ b/docs/standard.md @@ -143,7 +143,7 @@ record); verifier/reward.txt and score.json | Units | Run `-j` at a time, each with its own workspace (a git repository of its own), home and relay | | A relay per unit | Its record is the trial's `.work/relay`; the judge and simulated user are routes on it (`-judge`, `-simulator`). An option whose driver embeds gets a relay of the run's own before any unit, recorded in `.work/relay-benchmark` as the benchmark's cost. A 200 with no reply, not streamed, reaches the caller as a 502 (or the 429 or 402 its error held); the record keeps `answered: 200`. A call refused with 400, 413 or 422 records its inline images' types and sizes as `images`, never their bytes | | Job name | One run at a time per job name: another is refused before it writes anything. A later run under the same name moves each trial's earlier record aside to the trial's `.work/earlier.` | -| Backends (`--backend`) | `local`: processes on this machine. `container`: each unit's agent or capability in a container, its relay beside it (`standard/container.py`), on Docker, or on Podman 4.7 or later where `docker` is missing or is podman under that name. Podman needs no Docker API socket and no `containers.conf` change. Where a `docker` program is on the PATH, Nullius runs it even when it does not answer. The reason it gives then says that podman is on the machine. `docker`, the old name of `container`, is still accepted. For agent and capability runs the driver stays on the machine; Original setups instead containerize the driver and its answerer (`standard/setup_container.py`); in a container a check that runs what the agent left (the workspace kit's test) runs it in a throwaway, network-less container of the unit's image (`container.check_argv`), and no check reads through a link the agent planted; `config.json` records the backend and the image. `boat`: each unit's agent or capability in a boat.dev VM of its own (`standard/boat.py`, [The boat backend](#the-boat-backend)) | +| Backends (`--backend`) | `local`: processes on this machine. `container`: each unit's agent or capability in a container, its relay beside it (`standard/container.py`), on Docker, or on Podman 4.7 or later where `docker` is missing, is podman under that name, or does not answer (`docker ps` fails) while `podman ps` succeeds. Podman needs no Docker API socket and no `containers.conf` change. Where neither answers, Nullius keeps `docker` and its reason says that podman is on the machine. A host is probed by the same rule. `docker`, the old name of `container`, is still accepted. For agent and capability runs the driver stays on the machine; Original setups instead containerize the driver and its answerer (`standard/setup_container.py`); in a container a check that runs what the agent left (the workspace kit's test) runs it in a throwaway, network-less container of the unit's image (`container.check_argv`), and no check reads through a link the agent planted; `config.json` records the backend and the image. `boat`: each unit's agent or capability in a boat.dev VM of its own (`standard/boat.py`, [The boat backend](#the-boat-backend)) | | Progress | One line per stage on stderr and in `progress.jsonl`: `[nullius] stage=install agent=codex@2.2.2 state=done seconds=41 at=...` (`start`, `done`, `cached`, `failed`; a download's bytes every 5 s) for the base image, an environment, the upstream, a dataset, `driver.install` and the agent; then `stage=units n=10`, `stage=unit i=3 of=10 unit=... outcome=passed`, `stage=done`; `stage=refused` alone for a run refused first. From `stage=units` the job also writes `events.jsonl`, Harbor's live feed | | Outcomes | A last model call that got a 429 is `rate_limited` (calls in the order they began; a side model's later 200 does not cover the agent's own), a refused key an API error, both outside the agent's rate, unless the unit passed; a unit asked to stop (a `stop` beside its relay's records) keeps the score its driver gave despite the agent's own last 429; the agent's own last call over the model's context window is `ContextWindowExceededError`, the agent's; an agent past its limit for the unit, every turn included, is the agent's timeout ([time limits](cli/run.md#time-limits)). `"kind": "score"` leaves a score and no pass | | Rate limits | After a `rate_limited` end no unit starts for 60 s, or the provider's `Retry-After`, doubling to 300 s while units keep ending so | From dd247e4d4d9bf435dede06d6499e184fbdba021e Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Thu, 19 Nov 2026 16:25:07 +0100 Subject: [PATCH 2/3] feat(settings): say the other engine in Sandbox when both are installed --- app/nullius/sandbox.py | 32 +++++++++++++++++++++++----- app/static/settings/sandbox.js | 3 +++ app/templates/settings/settings.html | 2 +- app/tests/test_standard_container.py | 14 ++++++++++++ 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/app/nullius/sandbox.py b/app/nullius/sandbox.py index a491268f..399bbe28 100644 --- a/app/nullius/sandbox.py +++ b/app/nullius/sandbox.py @@ -19,6 +19,7 @@ import json import os import re +import shutil import subprocess from typing import List, Optional @@ -32,12 +33,33 @@ def docker_state() -> dict: """{ok, version} when the engine answers - its name and version, ``podman - 6.1.4`` - else {ok: False, error}.""" + 6.1.4`` - else {ok: False, error}; with ``also``, the other engine when both + are installed (``_other``), so the page says why this one is in use.""" why = container.unusable() - if why: - return {"ok": False, "error": why} - r = container._docker(["version", "--format", "{{.Server.Version}}"], timeout=30, check=False) - return {"ok": True, "version": f"{container.DOCKER} {r.stdout.strip()}"} + state = {"ok": False, "error": why} if why else None + if state is None: + r = container._docker(["version", "--format", "{{.Server.Version}}"], timeout=30, check=False) + state = {"ok": True, "version": f"{container.DOCKER} {r.stdout.strip()}"} + also = _other() + return dict(state, also=also) if also else state + + +def _other() -> Optional[dict]: + """The engine not in use, when it is installed too and is not the same program + (podman's ``docker``): {name, ok, said}, ``said`` its version or why it does not answer.""" + other = "podman" if container.DOCKER == "docker" else "docker" + path = shutil.which(other) + if not path or (other == "docker" and container._runs_podman(path)): + return None + try: + r = subprocess.run([other, "version", "--format", "{{.Server.Version}}"], stdin=subprocess.DEVNULL, + capture_output=True, text=True, timeout=10) + except (OSError, subprocess.SubprocessError) as e: + return {"name": other, "ok": False, "said": str(e)[:200]} + if r.returncode == 0 and r.stdout.strip(): + return {"name": other, "ok": True, "said": f"{other} {r.stdout.strip()}"} + lines = (r.stderr or r.stdout).strip().splitlines() + return {"name": other, "ok": False, "said": lines[-1][:200] if lines else f"exit {r.returncode}"} def _config_dir() -> str: diff --git a/app/static/settings/sandbox.js b/app/static/settings/sandbox.js index 465a1993..b8b9caab 100644 --- a/app/static/settings/sandbox.js +++ b/app/static/settings/sandbox.js @@ -30,6 +30,9 @@ const docker = $('[data-docker-state]'); docker.textContent = j.docker.ok ? '✓ ' + j.docker.version : '✕ ' + j.docker.error; docker.className = 'st-row-value ' + (j.docker.ok ? 'ok' : 'bad'); + const also = j.docker.also, other = $('[data-docker-also]'); + other.hidden = !also; + if (also) other.lastElementChild.textContent = also.ok ? also.said + ' is installed too' : also.name + ' is installed and does not answer: ' + also.said; boat = j.boat || {set: false, env: false}; render(); }; diff --git a/app/templates/settings/settings.html b/app/templates/settings/settings.html index 36eca7e0..c1e2f980 100644 --- a/app/templates/settings/settings.html +++ b/app/templates/settings/settings.html @@ -395,7 +395,7 @@

Judges

Sandbox

Where agents run, apart from this machine.

-
EngineChecking…
+
EngineChecking…

Boat

Checking…

{% if writable %}{% endif %}
diff --git a/app/tests/test_standard_container.py b/app/tests/test_standard_container.py index bd6628a6..bb301697 100644 --- a/app/tests/test_standard_container.py +++ b/app/tests/test_standard_container.py @@ -84,6 +84,20 @@ def test_with_both_on_the_path_the_one_that_answers_is_the_engine(self): with self.subTest(want=want): self.assertEqual(self.engine(), want) + def test_the_sandbox_says_the_other_engine_when_both_are_installed(self): + from nullius import sandbox + def other(engine): + with mock.patch.dict(os.environ, {"PATH": self.tmp}), mock.patch.object(container, "DOCKER", engine): + return sandbox._other() + self.program("podman", "#!/bin/sh\necho 5.4.2\n") + self.assertIsNone(other("podman")) # podman alone: nothing else to say + self.program("docker", "#!/bin/sh\necho 'Cannot connect to the Docker daemon at unix:///var/run/docker.sock' >&2\nexit 1\n") + self.assertEqual(other("podman"), {"name": "docker", "ok": False, + "said": "Cannot connect to the Docker daemon at unix:///var/run/docker.sock"}) + self.assertEqual(other("docker"), {"name": "podman", "ok": True, "said": "podman 5.4.2"}) + self.program("docker", "#!/bin/sh\nexec podman \"$@\"\n") # podman-docker's docker: the same program + self.assertIsNone(other("podman")) + def test_the_host_probe_and_the_run_there_pick_the_same_engine(self): from nullius import hosts for tool in ("sh", "sed", "grep", "hostname"): From 6055ce2b4ef20ae37c35bc5b0554cb9bf8a00a4d Mon Sep 17 00:00:00 2001 From: Harenome Ranaivoarivony-Razanajato Date: Thu, 19 Nov 2026 16:39:47 +0100 Subject: [PATCH 3/3] test(settings): name the engine with no other one on the PATH --- app/tests/test_sandbox.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/tests/test_sandbox.py b/app/tests/test_sandbox.py index 1baa8207..8342bb2c 100644 --- a/app/tests/test_sandbox.py +++ b/app/tests/test_sandbox.py @@ -54,7 +54,8 @@ def test_podman_logs_in_to_docker_hub_by_name_and_says_who(self): def test_the_engine_is_named_with_its_version(self): with mock.patch.object(container, "DOCKER", "podman"), mock.patch.object(container, "unusable", return_value=None), \ - mock.patch.object(container, "_docker", return_value=done(out="6.1.4\n")): + mock.patch.object(container, "_docker", return_value=done(out="6.1.4\n")), \ + mock.patch.object(sandbox.shutil, "which", return_value=None): # no other engine: the machine's own is not asked self.assertEqual(sandbox.docker_state(), {"ok": True, "version": "podman 6.1.4"}) def test_a_bad_username_or_empty_token_asks_docker_nothing(self):