Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions app/nullius/workspace.py
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,24 @@ def _head(job_dir: str) -> str:
return "\n".join(parts)


def heads(roots) -> tuple:
"""``_head`` of each of *roots* and of each entry in it: what
``parse.find_job_dirs`` over them, and the parse of an ended job found
there, can change by. A page's summary of ended jobs is kept while this is.
shortcut: a change inside a trial folder alone (a trial's result.json, or
the first one of a job without events.jsonl) or a link retargeted is not
seen; key on ``signature`` of each job if those must show."""
out = []
for root in roots:
try:
names = sorted(os.listdir(root))
except OSError:
out.append((root, None))
continue
out.append((root, _head(root), tuple((n, _head(os.path.join(root, n))) for n in names)))
return tuple(out)


def _ended(job: parse.Job) -> bool:
"""Its result records a finish and nothing running or pending: its trials are final."""
return job.finished_at is not None and job.state == "finished"
Expand Down
14 changes: 14 additions & 0 deletions app/tests/test_workspace.py
Original file line number Diff line number Diff line change
Expand Up @@ -383,6 +383,20 @@ def test_a_job_no_run_writes_is_read_by_its_head(self):
again, from_cache = workspace.load_job(self.ws, self.job_dir, runs={"job-b": None})
self.assertIs(again, first); self.assertTrue(from_cache)

def test_heads_move_with_a_job_added_removed_or_rewritten_not_with_a_trial(self):
before = workspace.heads([self.root])
_write_json(os.path.join(self.job_dir, "tau3-bench__t0__abc", "result.json"), {"rewritten": True})
self.assertEqual(workspace.heads([self.root]), before) # the shortcut's limit
_write_json(os.path.join(self.job_dir, "result.json"), {"id": "j1", "stats": {"n_running_trials": 1}})
rewritten = workspace.heads([self.root])
self.assertNotEqual(rewritten, before)
_job_dir(self.root, "job-b")
self.assertNotEqual(workspace.heads([self.root]), rewritten)
shutil.rmtree(os.path.join(self.root, "job-b"))
self.assertNotEqual(workspace.heads([self.root]), rewritten) # the root's own mtime moved
gone = os.path.join(self.root, "gone")
self.assertEqual(workspace.heads([gone]), ((gone, None),))

def test_index_lists_runs_without_a_parse(self):
self.assertEqual(workspace.job_index(self.ws), {})
workspace.load_job(self.ws, self.job_dir)
Expand Down
36 changes: 36 additions & 0 deletions app/ui_tests/test_run_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -975,6 +975,22 @@ def test_runs_and_checks_are_counted_alike_on_both_pages(self):
self.assertEqual([self.cells(rows[key])[c] for c in ("runs", "checks")], ["1", "0"]) # the merge stands for its two shards
self.assertEqual([self.cells(rows[f"run:{refused.id}"])[c] for c in ("runs", "checks", "state")], ["1", "1", "refused"])

def test_launches_kept_between_loads_follow_the_disk(self):
from tests.test_feasibility import write_job
rec = self.record("r3", "running")
said = lambda: [self.cells(self.rows()[f"run:{rec.id}"])[c] for c in ("runs", "checks", "state")]
self.assertEqual(said(), ["1", "1", "running"])
rec.state = "done" # a run ending
self.assertEqual(said(), ["1", "1", "finished"])
self.assertIn("last", web.runs._EVERY) # kept: every job has ended
write_job(rec.jobs, "r3-nop", "my-agent", {"a": 1}) # a job added: no check after all
self.assertEqual(said(), ["2", "0", "finished"])
shutil.rmtree(os.path.join(rec.jobs, "r3-nop")) # and removed: named, not written
self.assertEqual(said(), ["1", "1", "finished"])
cfg = os.path.join(rec.jobs, "r3-a", "config.json") # a job rewritten in place
self._write_json(cfg, {**json.loads(Path(cfg).read_text()), "agents": [{"name": "oracle"}]})
self.assertEqual(said(), ["0", "2", "finished"])

def test_no_launch_is_one_line(self):
execution.EXECUTIONS.clear()
self.assertIn('<div class="alert">No launch yet.</div>', self.get("/launches").get_data(as_text=True))
Expand Down Expand Up @@ -1140,6 +1156,26 @@ def test_runs_and_checks_count_as_the_runs_and_launches_pages_count_them(self):
ask = self.get(f"/workspaces/{self.ws.name}/roots/ask?root={d / 'jobs'}").get_data(as_text=True)
self.assertIn("<p>1 run · 2 checks · ", ask) # the folder's Delete counts its run alike

def test_counts_kept_between_loads_follow_the_disk(self):
from tests.test_feasibility import write_job
other, jobs = self.other(), str(self.run_dir / "jobs")
counts = lambda: (self.cell(other, 2), self.cell(other, 3), self.cell(other, 5))
self.assertEqual(counts(), ("2", "0", "1"))
self.assertIn(other.name, web.workspaces._COUNTS) # kept: its jobs have ended
self.assertEqual(counts(), ("2", "0", "1"))
write_job(jobs, "r1-c", "nop", {"a": 0}) # a job added
self.assertEqual(counts(), ("2", "1", "1"))
shutil.rmtree(os.path.join(jobs, "r1-c")) # and removed
self.assertEqual(counts(), ("2", "0", "1"))
cfg = os.path.join(jobs, "r1-b", "config.json") # a job rewritten in place
self._write_json(cfg, {**json.loads(Path(cfg).read_text()), "agents": [{"name": "oracle"}]})
self.assertEqual(counts(), ("1", "1", "1"))
rec = LaunchesPage.record(self, "r9") # a launch added to it: its roots change
other.roots.append(rec.jobs); workspace.save(other)
self.assertEqual(counts(), ("2", "2", "2"))
del execution.EXECUTIONS["r9"] # the run records change
self.assertEqual(counts(), ("2", "1", "1"))

def test_launches_count_as_the_filtered_launches_page(self):
bare = workspace.create(self.home, [str(self.tmp / "none")], label="bare")
rows = re.findall(r'<tr data-launch="[^"]+".*?data-f-ws="([^"]*)"', self.get("/launches").get_data(as_text=True))
Expand Down
18 changes: 16 additions & 2 deletions app/web/runs.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,12 +132,26 @@ def _record(ex, merged=()) -> tuple:
def _every() -> tuple:
"""Every launch Nullius made, newest first: every run record, grouped as
the Runs page groups them, each record's jobs read without their trials,
and a merge of them written beside them. Returns (job id -> job, launches)."""
and a merge of them written beside them. Returns (job id -> job, launches).
Kept while the records' states and their folders' heads are unchanged and
every job had ended (``workspace.heads``)."""
recs = list(execution.EXECUTIONS.values())
key = (tuple((e.id, e.state, e.jobs) for e in recs), workspace.heads(dict.fromkeys(e.jobs for e in recs)))
last = _EVERY.get("last")
if last and last[0] == key:
return last[1]
named = {str(x.get("name")) for e in recs for x in e.plan.jobs}
jobs = [parse.parse_job(d, trials=False) for d in parse.find_job_dirs([e.jobs for e in recs if os.path.isdir(e.jobs)])]
jobs = [j for j in jobs if j.job_id in named or named & set(getattr(j, "merged_from", []))]
return {j.job_id: j for j in jobs}, launches.group(jobs, {e.id: _record(e) for e in recs})
every = {j.job_id: j for j in jobs}, launches.group(jobs, {e.id: _record(e) for e in recs})
if all(j.state == "finished" for j in jobs):
_EVERY["last"] = (key, every)
else:
_EVERY.pop("last", None)
return every


_EVERY: dict = {} # "last": (key, _every()) of the last call that found every job ended


def _tally(g, by_id) -> dict:
Expand Down
23 changes: 19 additions & 4 deletions app/web/workspaces.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,24 +61,39 @@ def _root(path, launched, memo) -> dict:
return {**memo[real], "path": path}


def _counts(ws) -> dict:
def _counts(ws, recs) -> dict:
"""The runs and the checks (nop, oracle) of *ws*, summed over the launches
its Runs page shows, each counted as there and on the Launches page
(``runs._tally``). Its roots are walked, each unchanged job read from its
cache."""
cache. A closed workspace whose jobs had all ended keeps its counts while
its roots, their heads (``workspace.heads``) and ``recs``, the run records'
ids and states, are unchanged."""
is_open = shared.STATE["workspace"] is not None and shared.STATE["workspace"].name == ws.name
key = None if is_open else (tuple(ws.roots), tuple(ws.job_roots), workspace.heads(ws.job_roots or ws.roots), recs)
last = _COUNTS.get(ws.name)
if key is not None and last and last[0] == key:
return last[1]
jobs, _, _, groups = runs_web._grouped(None if is_open else ws)
by_id = {j.job_id: j for j in jobs}
tallies = [runs_web._tally(g, by_id) for g in groups]
return {"runs": sum(t["n_runs"] for t in tallies), "checks": sum(t["n_checks"] for t in tallies)}
counts = {"runs": sum(t["n_runs"] for t in tallies), "checks": sum(t["n_checks"] for t in tallies)}
if key is not None and all(j.state == "finished" for j in jobs):
_COUNTS[ws.name] = (key, counts)
else:
_COUNTS.pop(ws.name, None)
return counts


_COUNTS: dict = {} # workspace name -> (key, _counts) of a workspace whose jobs had all ended


@app.route("/workspaces")
def workspaces_page():
wss, launched = workspace.list_all(shared.WS_HOME), runs_web._held_launches()
held = [w.name for _, _, holders in launched[0] for w in holders]
recs = tuple((e.id, e.state) for e in execution.EXECUTIONS.values())
return render_template("workspaces/workspaces.html", workspaces=wss, csrf=shared._csrf_token(),
counts={ws.name: _counts(ws) for ws in wss}, n_judged={ws.name: store.count(ws.dir) for ws in wss}, n_launches={ws.name: held.count(ws.name) for ws in wss})
counts={ws.name: _counts(ws, recs) for ws in wss}, n_judged={ws.name: store.count(ws.dir) for ws in wss}, n_launches={ws.name: held.count(ws.name) for ws in wss})


@app.route("/workspaces/open/<name>", methods=["POST"])
Expand Down
Loading