Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions agentdescent/artifact_snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -426,8 +426,13 @@ def evaluate_snapshot_spec(
kind = snapshot.kind or spec.kind
with tempfile.TemporaryDirectory(prefix="ad-snapshot-eval-") as tmp:
target = _write_target(snapshot, kind, tmp)
# Staging stays at tmp/artifact (#1150). The dest agent's layout
# prefix comes from EvolveSpec.artifact_id(), which prefers `name`
# over the target basename, so a skill_dir snapshot is visible at
# `.claude/skills/<artifact_id>/` rather than `.../artifact/` (#1164).
dest = replace(
spec, target=target,
name=snapshot.artifact_id,
editable=tuple(snapshot.editable),
frozen=tuple(snapshot.frozen),
)
Expand Down
6 changes: 6 additions & 0 deletions changelog.d/1164-snapshot-eval-skill-name.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Fixed
`snapshot eval` of a `skill_dir` snapshot now exposes the candidate under
the snapshot's `artifact_id` in the destination layout (for example
`.claude/skills/csv-total/`), so a demo export re-evaluated with `--spec`
scores the learned skill instead of 0 because the dest agent looked under
`.claude/skills/artifact/` (#1164).
56 changes: 56 additions & 0 deletions tests/test_snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,62 @@ def boom(*args, **kwargs):
evaluate_snapshot_spec(snap, other)


def test_evaluate_snapshot_spec_puts_a_skill_dir_under_its_artifact_id():
"""The dest agent reads `.claude/skills/<artifact_id>/`, not `.../artifact/` (#1164).

Staging still uses the fixed `tmp/artifact` path (#1150). The destination
spec's `name` carries the snapshot's id so `artifact_id()` and the
`claude_skill` prefix become `.claude/skills/csv-total/`.
"""
from agentdescent.demo import DEMO_SKILL, _SKILL_MD
from agentdescent.runners import tree_runner

csv_body = "id,amount\n1,100\n2,200\n3,21\n" # amount=321, id=6
dest = EvolveSpec(
kind="skill_dir",
target="ignored-target",
data={"inline": [
{"id": "t0", "prompt": "What is the total?", "gold": "321",
"fixtures": {"data.csv": csv_body}},
]},
score="exact",
agent={"ref": "agentdescent.demo:offline_agent"},
prompt_template="{prompt}",
)

def _snap(rules):
return ArtifactSnapshot(
artifact_id=DEMO_SKILL, kind="skill_dir",
files={"SKILL.md": _SKILL_MD, "references/rules.md": rules},
)

recorded = {}
real_tree_runner = tree_runner

def spy(agent, layout="claude_skill", **kwargs):
recorded["name"] = kwargs.get("name")
recorded["layout"] = layout
return real_tree_runner(agent, layout=layout, **kwargs)

monkey = pytest.MonkeyPatch()
monkey.setattr("agentdescent.evolvespec.tree_runner", spy)
try:
learned = evaluate_snapshot_spec(_snap("COLUMN: amount\n"), dest)
seed = evaluate_snapshot_spec(_snap("COLUMN: id\n"), dest)
finally:
monkey.undo()

assert recorded["name"] == DEMO_SKILL
assert recorded["layout"] == "claude_skill"
assert learned.failed == ()
assert learned.n == 1 and learned.reward == 1.0
assert learned.searched is False
assert seed.failed == ()
assert seed.n == 1 and seed.reward == 0.0
# Staging itself stays under the fixed name; this test only pins the
# layout prefix the dest agent sees.


def test_plugin_kind_snapshot_is_the_plugin_tree_not_a_host():
files = {"plugin.py": "print('plugin')\n", "README.md": "a plugin\n"}
snap = ArtifactSnapshot.from_filetree(
Expand Down
Loading