diff --git a/agentdescent/artifact_snapshot.py b/agentdescent/artifact_snapshot.py index fb21e1dc..383e3314 100644 --- a/agentdescent/artifact_snapshot.py +++ b/agentdescent/artifact_snapshot.py @@ -426,8 +426,13 @@ def evaluate_snapshot_spec( kind = snapshot.kind or spec.kind with tempfile.TemporaryDirectory(prefix="ad-snapshot-eval-") as tmp: target = _write_target(snapshot, kind, tmp) + # Staging stays at tmp/artifact (#1150). The dest agent's layout + # prefix comes from EvolveSpec.artifact_id(), which prefers `name` + # over the target basename, so a skill_dir snapshot is visible at + # `.claude/skills//` rather than `.../artifact/` (#1164). dest = replace( spec, target=target, + name=snapshot.artifact_id, editable=tuple(snapshot.editable), frozen=tuple(snapshot.frozen), ) diff --git a/changelog.d/1164-snapshot-eval-skill-name.md b/changelog.d/1164-snapshot-eval-skill-name.md new file mode 100644 index 00000000..8c79b68b --- /dev/null +++ b/changelog.d/1164-snapshot-eval-skill-name.md @@ -0,0 +1,6 @@ +Fixed +`snapshot eval` of a `skill_dir` snapshot now exposes the candidate under +the snapshot's `artifact_id` in the destination layout (for example +`.claude/skills/csv-total/`), so a demo export re-evaluated with `--spec` +scores the learned skill instead of 0 because the dest agent looked under +`.claude/skills/artifact/` (#1164). diff --git a/tests/test_snapshot.py b/tests/test_snapshot.py index 79ab5a58..0f8055e7 100644 --- a/tests/test_snapshot.py +++ b/tests/test_snapshot.py @@ -180,6 +180,62 @@ def boom(*args, **kwargs): evaluate_snapshot_spec(snap, other) +def test_evaluate_snapshot_spec_puts_a_skill_dir_under_its_artifact_id(): + """The dest agent reads `.claude/skills//`, not `.../artifact/` (#1164). + + Staging still uses the fixed `tmp/artifact` path (#1150). The destination + spec's `name` carries the snapshot's id so `artifact_id()` and the + `claude_skill` prefix become `.claude/skills/csv-total/`. + """ + from agentdescent.demo import DEMO_SKILL, _SKILL_MD + from agentdescent.runners import tree_runner + + csv_body = "id,amount\n1,100\n2,200\n3,21\n" # amount=321, id=6 + dest = EvolveSpec( + kind="skill_dir", + target="ignored-target", + data={"inline": [ + {"id": "t0", "prompt": "What is the total?", "gold": "321", + "fixtures": {"data.csv": csv_body}}, + ]}, + score="exact", + agent={"ref": "agentdescent.demo:offline_agent"}, + prompt_template="{prompt}", + ) + + def _snap(rules): + return ArtifactSnapshot( + artifact_id=DEMO_SKILL, kind="skill_dir", + files={"SKILL.md": _SKILL_MD, "references/rules.md": rules}, + ) + + recorded = {} + real_tree_runner = tree_runner + + def spy(agent, layout="claude_skill", **kwargs): + recorded["name"] = kwargs.get("name") + recorded["layout"] = layout + return real_tree_runner(agent, layout=layout, **kwargs) + + monkey = pytest.MonkeyPatch() + monkey.setattr("agentdescent.evolvespec.tree_runner", spy) + try: + learned = evaluate_snapshot_spec(_snap("COLUMN: amount\n"), dest) + seed = evaluate_snapshot_spec(_snap("COLUMN: id\n"), dest) + finally: + monkey.undo() + + assert recorded["name"] == DEMO_SKILL + assert recorded["layout"] == "claude_skill" + assert learned.failed == () + assert learned.n == 1 and learned.reward == 1.0 + assert learned.searched is False + assert seed.failed == () + assert seed.n == 1 and seed.reward == 0.0 + # Staging itself stays under the fixed name; this test only pins the + # layout prefix the dest agent sees. + + def test_plugin_kind_snapshot_is_the_plugin_tree_not_a_host(): files = {"plugin.py": "print('plugin')\n", "README.md": "a plugin\n"} snap = ArtifactSnapshot.from_filetree(