From dd1f8de6a775d4b825a9bd015a7b0284269635bb Mon Sep 17 00:00:00 2001 From: Eirik Botten Nicolaysen Date: Fri, 6 Nov 2026 20:50:36 +0100 Subject: [PATCH 1/3] Test that each run's meta survives a repeated-results round trip dev already passes meta=run_data.get("meta") in RepeatedExperimentResults.load (584f120). This keeps the regression test from the original fix, which also checks that a run saved without meta loads with an empty dict. --- tests/test_repeated_experiments.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_repeated_experiments.py b/tests/test_repeated_experiments.py index a2f3d12..c01dd0a 100644 --- a/tests/test_repeated_experiments.py +++ b/tests/test_repeated_experiments.py @@ -569,6 +569,20 @@ def test_save_load_preserves_judge_metadata(self, tmp_path): assert loaded._judge == judge assert loaded.to_dict()["judge"]["judge_model"] == "judge-x" + def test_save_load_preserves_each_runs_meta(self, tmp_path): + """Each run's meta survives save()/load(), as AuditResults.load keeps it.""" + meta = {"judge_model": "judge-x", "target_model": "m", "scenario_pack": "safety"} + run = AuditResults(_make_results(["pass"]).results, meta=meta) + results = RepeatedExperimentResults({"m": [run, _make_results(["low"])]}) + + path = str(tmp_path / "exp.json") + results.save(path) + loaded = RepeatedExperimentResults.load(path) + + assert loaded._runs["m"][0].meta == meta + assert loaded._runs["m"][1].meta == {} + assert loaded.to_dict()["runs"]["m"][0]["meta"] == meta + # --------------------------------------------------------------------------- # AuditExperiment — on_model_done callback From d2283043a8e97200894c167ae01f6191abd16c35 Mon Sep 17 00:00:00 2001 From: Eirik Botten Nicolaysen Date: Fri, 6 Nov 2026 20:50:44 +0100 Subject: [PATCH 2/3] Test that AuditExperiment records the scenario pack in each run's meta dev already sets result.meta["scenario_pack"] when AuditExperiment is given a pack name (584f120). This keeps the regression test from the original fix, which also checks that a run given a scenario list records none. --- tests/test_repeated_experiments.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_repeated_experiments.py b/tests/test_repeated_experiments.py index c01dd0a..f58f18a 100644 --- a/tests/test_repeated_experiments.py +++ b/tests/test_repeated_experiments.py @@ -86,6 +86,24 @@ def test_n_repetitions_3_produces_3_runs(self): assert isinstance(results, RepeatedExperimentResults) assert len(results._runs["test-model"]) == 3 + def test_pack_name_is_recorded_in_each_runs_meta(self): + """A run started with a pack name records it, as ModelAuditor.run does.""" + + async def fake_run_async(self_a, scenarios, max_turns=None, language="English", **kwargs): + return AuditResults( + _make_results(["pass"]).results, + meta=self_a._run_meta(scenarios, max_turns, language), + ) + + exp = _make_experiment(n_repetitions=2) + with patch.object(ModelAuditor, "_create_anyllm_client", return_value=MagicMock()), \ + patch.object(ModelAuditor, "run_async", new=fake_run_async): + from_pack = asyncio.run(exp.run_async(scenarios="safety")) + from_list = asyncio.run(exp.run_async(scenarios=SCENARIOS)) + + assert [run.meta.get("scenario_pack") for run in from_pack._runs["test-model"]] == ["safety", "safety"] + assert all("scenario_pack" not in run.meta for run in from_list._runs["test-model"]) + def test_n_repetitions_1_is_default_compatible(self): exp = _make_experiment(n_repetitions=1) r = _make_results(["low"]) From 0919c58675ff1a7ae82d21ffa3365a6d903116e1 Mon Sep 17 00:00:00 2001 From: Eirik Botten Nicolaysen Date: Tue, 3 Nov 2026 17:51:50 +0100 Subject: [PATCH 3/3] Show a missing score in the README results example AuditResults.score is None when nothing was graded, so the example printed "Score: None/100". It now prints the same "n/a (nothing graded)" that summary() prints in that case. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2312df9..28770a8 100644 --- a/README.md +++ b/README.md @@ -1331,7 +1331,8 @@ Two things to know: results = auditor.run("safety") # Summary statistics -print(f"Score: {results.score}/100") +score = "n/a (nothing graded)" if results.score is None else f"{results.score}/100" +print(f"Score: {score}") print(f"Passed: {results.passed}/{len(results)}") print(f"Failed: {results.failed}") # graded anything other than pass print(f"Errors: {results.errors}") # never graded: API failure or unreadable judge reply