From 2c7b745ae69ead35f9078bc11eb8c0d70b6bff6f Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 23 Oct 2026 14:20:02 -0700 Subject: [PATCH 01/10] feat(validate): deploy the mail-mcp example's env, named by an env alias, in place of an internal env The validator deployed multi-caf64d, an env only Scale's stores hold, so anyone else's validate failed its MCP check. It now deploys mail-mcp: an example MCP env with the search_emails tool the check calls, named by the alias its bundle's aliases.toml gives it, which now takes [envs] as it takes [agents]. validate writes the example's env into local stores when the store has none of that id; a store's own env of that id wins, which is how a service gets it. Co-Authored-By: Claude Opus 5.5 (1M context) --- AGENTS.md | 6 ++ pyproject.toml | 1 + src/agent_env/a2a_agent/a2a_agent.py | 14 +-- src/agent_env/agents/AGENTS.md | 7 +- src/agent_env/bundle/__init__.py | 3 +- src/agent_env/bundle/parse.py | 47 ++++---- src/agent_env/bundle/plan.py | 100 +++++++++++------- src/agent_env/bundle/run.py | 52 ++++++--- src/agent_env/cli/a2a_agent/validate.py | 34 +++++- src/agent_env/cli/plugin.py | 12 +-- src/agent_env/cli/run.py | 6 +- src/agent_env/env/env.py | 46 +++++++- src/agent_env/examples/mail-mcp/README.md | 10 ++ src/agent_env/examples/mail-mcp/aliases.toml | 3 + .../mail-mcp/artifacts/check_mail/verify.py | 15 +++ .../examples/mail-mcp/envs/mail/Dockerfile | 8 ++ .../mail-mcp/envs/mail/requirements.txt | 3 + .../examples/mail-mcp/envs/mail/server.py | 32 ++++++ .../examples/mail-mcp/tasks/smoke.json | 5 + src/agent_env/plugins/_bundles.py | 95 +++++++++-------- .../cli/run_bundle_built_envs_test.py | 18 ++++ tst/unit/bundle/aliases_test.py | 41 +++++-- tst/unit/bundle/installed_entities_test.py | 34 ++++++ tst/unit/bundle/installed_test.py | 3 +- tst/unit/bundle/preflight_test.py | 21 +++- tst/unit/bundle/shipped_bundles_test.py | 18 +++- tst/unit/cli/a2a_agent_validate_test.py | 72 +++++++++++++ tst/unit/env/alias_get_test.py | 94 ++++++++++++++++ tst/unit/test_clean_install_wheel.py | 2 +- tst/unit/test_release_protocol_pins.py | 1 + 30 files changed, 646 insertions(+), 157 deletions(-) create mode 100644 src/agent_env/examples/mail-mcp/README.md create mode 100644 src/agent_env/examples/mail-mcp/aliases.toml create mode 100644 src/agent_env/examples/mail-mcp/artifacts/check_mail/verify.py create mode 100644 src/agent_env/examples/mail-mcp/envs/mail/Dockerfile create mode 100644 src/agent_env/examples/mail-mcp/envs/mail/requirements.txt create mode 100644 src/agent_env/examples/mail-mcp/envs/mail/server.py create mode 100644 src/agent_env/examples/mail-mcp/tasks/smoke.json create mode 100644 tst/unit/cli/a2a_agent_validate_test.py create mode 100644 tst/unit/env/alias_get_test.py diff --git a/AGENTS.md b/AGENTS.md index 713d7ec9..6097587e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -158,6 +158,12 @@ Out-of-tree code plugs in either through `config.toml`, as a `module:Class` poin the solver for one run (each step deploying or installing an agent a `prompt_agent` step prompts, not its user simulator: `plan.solver_steps`), pulled in and checked as a task's reference to it would be, through per-step `step_params` overrides, so the tasks stay as written. +- `aliases.toml` gives envs aliases the same way (`[envs] = ["", ...]`; the `mail-mcp` example names + its env `mail-mcp`), resolved by the planner (`Plan.env_aliases`) and by `Env.get`. The env `Env.get` gives for an + alias answers to the alias's id, so a deployment records, and steps look it up by, the id the task names; an alias + names its env only where namespace routing is on (the CLI), so a service needs an env of the alias's id in its store. + `agent-env a2a-agent validate` deploys `mail-mcp` (`A2AAgent.VALIDATION_ENV_ID`), writing the example's env with + `bundle.write_bundle` (a bundle's writes and infra, no run) when the store has none of that id. - Environment providers have no table: an installed package registers one as an `agent_env.env_providers` entry point named its `type`, which records carry as `env_provider_type`. - `[plugins.]` is reserved for an installed plugin's own settings, read with diff --git a/pyproject.toml b/pyproject.toml index d0a4459f..ad70e95c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,6 +119,7 @@ agent-env = "agent_env.cli:cli" agents = "agent_env" hello = "agent_env.examples" echo-mcp = "agent_env.examples" +mail-mcp = "agent_env.examples" [project.urls] Homepage = "https://www.agentenvframework.com" diff --git a/src/agent_env/a2a_agent/a2a_agent.py b/src/agent_env/a2a_agent/a2a_agent.py index d9714f50..47b9b5ac 100644 --- a/src/agent_env/a2a_agent/a2a_agent.py +++ b/src/agent_env/a2a_agent/a2a_agent.py @@ -430,13 +430,13 @@ def get(cls, id: str, version: Optional[int] = None) -> A2AAgent: if named is None: raise NotFoundError(refusal) from None if version is not None: - raise NotFoundError(f"{id!r} is an alias of {named.agent_id}, which comes with {named.declared_by}, so " - "it has no fixed version; drop the pin") from None + raise NotFoundError(f"{id!r} is an alias of {named.target_id}, which comes with " + f"{named.declared_by}, so it has no fixed version; drop the pin") from None try: - return store.get(named.agent_id) + return store.get(named.target_id) except NotFoundError: - raise NotFoundError(f"{id!r} is an alias of {named.agent_id}, which comes with {named.declared_by} and isn't " - "in the store yet: an `agent-env run` of a bundle that uses it writes it") from None + raise NotFoundError(f"{id!r} is an alias of {named.target_id}, which comes with {named.declared_by} and " + "isn't in the store yet: an `agent-env run` of a bundle that uses it writes it") from None @classmethod def put(cls, **kwargs) -> A2AAgent: @@ -450,7 +450,9 @@ def query(cls) -> A2AAgentQuery: from .store import A2AAgentQuery, get_a2a_agent_store return A2AAgentQuery(get_a2a_agent_store()) - VALIDATION_ENV_ID: ClassVar[str] = "multi-caf64d" + # The env validate gives the agent it checks: the alias of the mail-mcp example's env, which a store's env of that + # id shadows. + VALIDATION_ENV_ID: ClassVar[str] = "mail-mcp" def update_metadata(self, new_metadata: dict[str, Any]) -> None: """Replace this agent's metadata in the database using compare-and-swap.""" diff --git a/src/agent_env/agents/AGENTS.md b/src/agent_env/agents/AGENTS.md index d574cf03..fbbb859c 100644 --- a/src/agent_env/agents/AGENTS.md +++ b/src/agent_env/agents/AGENTS.md @@ -228,9 +228,10 @@ agent-env a2a-agent validate --id my-agent # agent-env's built agent-env a2a-agent get --id my-agent # the validation results ``` -`validate` runs a fixed set of checks against the reference env `multi-caf64d`, which must exist in the document -store; the checks are in `src/agent_env/a2a_agent/validator.py` and -`src/agent_env/task_step/task_steps/a2a_agent_validator/`, one step each. It doesn't cover everything an agent can +`validate` runs a fixed set of checks against the reference env `mail-mcp`, an inbox with a `search_emails` tool: the +alias of the `mail-mcp` example's env (`src/agent_env/examples/mail-mcp/`), which `validate` writes into local stores +when the store has no env of that id, and which other stores need an env of that id for. The checks are in +`src/agent_env/a2a_agent/validator.py` and `src/agent_env/task_step/task_steps/a2a_agent_validator/`, one step each. It doesn't cover everything an agent can advertise: `triggers/v1`, `attribution-probe/v1`, host-mode `install/v1` and custom extensions need tests of your own. diff --git a/src/agent_env/bundle/__init__.py b/src/agent_env/bundle/__init__.py index 49ed558c..ddc1c99c 100644 --- a/src/agent_env/bundle/__init__.py +++ b/src/agent_env/bundle/__init__.py @@ -1,7 +1,7 @@ """Bundles: folders of entities that run without a registry.""" from .parse import Bundle, BundleEntry, BundleError, BundleKind, is_ignored, parse_bundle -from .run import BundleRun, DryRun, EvalRun, Outcome, RunInterrupted, TaskRun, dry_run_bundle, run_bundle +from .run import BundleRun, DryRun, EvalRun, Outcome, RunInterrupted, TaskRun, dry_run_bundle, run_bundle, write_bundle __all__ = [ "Bundle", @@ -18,4 +18,5 @@ "is_ignored", "parse_bundle", "run_bundle", + "write_bundle", ] diff --git a/src/agent_env/bundle/parse.py b/src/agent_env/bundle/parse.py index 4784e14b..f8d0ebbe 100644 --- a/src/agent_env/bundle/parse.py +++ b/src/agent_env/bundle/parse.py @@ -21,9 +21,9 @@ from pathlib import Path from typing import Any, NoReturn -from agent_env.bundle._fs import Unreadable, fold, home, on_disk, read_regular, show +from agent_env.bundle._fs import Unreadable, fold, home, on_disk, read_regular, show, with_article from agent_env.entity.refs import EntityKind -from agent_env.plugins._bundles import ALIASES_FILE, read_aliases +from agent_env.plugins._bundles import AGENT_ALIASES, ALIASES_FILE, ENV_ALIASES, read_aliases from agent_env.store.ids import LOCAL_PREFIX, MAX_AUTHORED_LOCAL_ID_BYTES, validate_local_id from agent_env.utils.paths import LEAVINGS, os_reason @@ -84,9 +84,9 @@ class Bundle: description: str | None # README.md's first line of text entries: tuple[BundleEntry, ...] ignored: tuple[Path, ...] # root-relative, as on disk: entries that are neither kinds nor entities - # The aliases its aliases.toml gives its agents, by agent name, which a reference names an agent by once the bundle - # is installed. - aliases: Mapping[str, tuple[str, ...]] = field(default_factory=dict) + # The aliases its aliases.toml gives its agents and envs, by table and then folder name, which a reference names + # one by once the bundle is installed. + aliases: Mapping[str, Mapping[str, tuple[str, ...]]] = field(default_factory=dict) class BundleError(ValueError): @@ -158,28 +158,31 @@ def parse(self) -> Bundle: aliases=aliases, ) - def _aliases(self, entries: list[BundleEntry]) -> dict[str, tuple[str, ...]]: - """The aliases the bundle's aliases.toml gives its agents. Each names an agent the bundle has, by its folder - name, so that agent keeps the id its folder gives it, and none is another of its agents' name or id.""" + def _aliases(self, entries: list[BundleEntry]) -> dict[str, dict[str, tuple[str, ...]]]: + """The aliases the bundle's aliases.toml gives its agents and envs. Each names one the bundle has, of its + table's kind, by its folder name, so it keeps the id its folder gives it, and none is another one's name or id.""" if self.aliases_file is None: return {} declared, problems = read_aliases(self.aliases_file) for problem in problems: self._problem(self.aliases_file, problem) - agents = {entry.name: entry for entry in entries if entry.kind is BundleKind.AGENT} - taken = {key: entry for entry in agents.values() for key in (entry.name, entry.id)} - for name, aliases in declared.items(): - agent = agents.get(name) - if agent is None: - self._problem(self.aliases_file, f"agents.{name}: this bundle has no agent {name!r}") - continue - if agent.id != f"{self.id_root}/{name}": - self._problem(self.aliases_file, f"agents.{name}: the agent declares the id {agent.id!r}, and an alias " - f"names an agent by its folder, so it must keep the id {self.id_root}/{name}") - for alias in aliases: - if (other := taken.get(alias)) is not None and other is not agent: - what = "name" if alias == other.name else "id" - self._problem(self.aliases_file, f"agents.{name}: {alias!r} is the agent {other.name!r}'s {what}") + for table, kind, noun in ((AGENT_ALIASES, BundleKind.AGENT, "agent"), (ENV_ALIASES, BundleKind.ENV, "env")): + own = {entry.name: entry for entry in entries if entry.kind is kind} + taken = {key: entry for entry in own.values() for key in (entry.name, entry.id)} + for name, aliases in declared.get(table, {}).items(): + entry = own.get(name) + if entry is None: + self._problem(self.aliases_file, f"{table}.{name}: this bundle has no {noun} {name!r}") + continue + if entry.id != f"{self.id_root}/{name}": + self._problem(self.aliases_file, f"{table}.{name}: the {noun} declares the id {entry.id!r}, and an " + f"alias names {with_article(noun)} by its folder, so it must keep the id " + f"{self.id_root}/{name}") + for alias in aliases: + if (other := taken.get(alias)) is not None and other is not entry: + what = "name" if alias == other.name else "id" + self._problem(self.aliases_file, + f"{table}.{name}: {alias!r} is the {noun} {other.name!r}'s {what}") return declared def _derive_id_root(self) -> str: diff --git a/src/agent_env/bundle/plan.py b/src/agent_env/bundle/plan.py index 5067eb6e..91c6ae16 100644 --- a/src/agent_env/bundle/plan.py +++ b/src/agent_env/bundle/plan.py @@ -39,8 +39,8 @@ from agent_env.env.envs.website import WebsiteEnv from agent_env.env.published_images import published_agent_image from agent_env.env.registry import get_env_registry -from agent_env.env.store import ENVS_COLLECTION -from agent_env.plugins._bundles import InstalledAliases, installed_aliases +from agent_env.env.store import ENVS_COLLECTION, get_env_store +from agent_env.plugins._bundles import ENV_ALIASES, InstalledAliases, installed_aliases from agent_env.providers.env_providers.constants import ( AGENT_ENV_WEBSITE_BACKEND_SUFFIX, AGENT_ENV_WEBSITE_FRONTEND_SUFFIX, @@ -79,7 +79,12 @@ def _stored_agent(id: str, version: int | None) -> A2AAgent: return get_a2a_agent_store().get(id, version) -STORE_GETTERS = {EntityKind.ENV: Env.get, EntityKind.AGENT: _stored_agent, EntityKind.ARTIFACT: Artifact.get, +def _stored_env(id: str, version: int | None) -> Env: + """The store's env of ``id`` itself, as ``_stored_agent`` is the store's agent.""" + return get_env_store().get(id, version) + + +STORE_GETTERS = {EntityKind.ENV: _stored_env, EntityKind.AGENT: _stored_agent, EntityKind.ARTIFACT: Artifact.get, EntityKind.TASK: Task.get} # The type a field naming an environment artifact takes: what an artifact referencing one holds. @@ -116,13 +121,14 @@ def unwritten(self, image_id: str) -> DockerImageArtifact: @dataclass(frozen=True) class AliasUse: - """An agent alias the run names, and the agent it names: the one an installed bundle gives the alias or, when the - store has an agent of that id, that one, which shadows it.""" + """An agent or env alias the run names, and what it names: the one an installed bundle gives the alias or, when + the store has one of that id, that one, which shadows it.""" alias: str - target_id: str # the agent the installed bundle gives the alias + target_id: str # the agent or env the installed bundle gives the alias declared_by: str # "'s bundle" - store_version: int | None = None # the store's agent of the alias's id, when it shadows the alias + store_version: int | None = None # the store's of the alias's id, when it shadows the alias + kind: EntityKind = EntityKind.AGENT @property def agent_id(self) -> str: @@ -165,6 +171,7 @@ class Plan: published: Mapping[str, Publication] = field(default_factory=dict) aliases: Mapping[str, AliasUse] = field(default_factory=dict) # each agent alias the run names, by alias agent: AgentOverride | None = None # the agent --agent names for the solver + env_aliases: Mapping[str, AliasUse] = field(default_factory=dict) # each env alias the run names, by alias def bundle_of(self, entry: BundleEntry) -> Bundle: """The bundle ``entry`` comes from: the installed one for an entry pulled in from one, else the run's.""" @@ -221,9 +228,12 @@ def __init__(self, resolved: ResolvedBundle, installed: Installed | None = None) self.others: tuple[InstalledBundle, ...] | None = None # the installed bundles but this one, once listed self.pullable: dict[str, _Pullable | str] = {} # each installed bundle read, by id root, or why it can't be self.unpulled: dict[tuple[EntityKind, str, int | None], str] = {} # why an installed bundle's id isn't pulled - self.alias_table: InstalledAliases | None = None # the installed aliases, once a plan reads them - self.alias_uses: dict[str, AliasUse] = {} # each alias the plan names, by alias + self.alias_table: InstalledAliases | None = None # the installed agent aliases, once a plan reads them + self.alias_uses: dict[str, AliasUse] = {} # each agent alias the plan names, by alias self.alias_stored: dict[str, int | None] = {} # the store's latest version of each alias's id, as read + self.env_alias_table: InstalledAliases | None = None + self.env_alias_uses: dict[str, AliasUse] = {} + self.env_alias_stored: dict[str, int | None] = {} self.pulling: set[_Key] = set() self.own = _by_id(resolved.bundle) # what a pulled entity's id names in this bundle images: dict[BundleEntry, list[BundleImage]] = {} @@ -277,38 +287,46 @@ def _default_agents(self, task: ResolvedEntry, agent_id: str | None = None) -> l # Agents named by an installed bundle's alias def _resolve_aliases(self) -> dict[str, AliasUse]: - """Point each task's reference to an agent by an alias an installed bundle gives it, and the configured default - when it's one, at that agent, pulled into the plan, unless the store has an agent of the alias's id, which the - reference keeps naming. A field naming the alias is rewritten to the agent's id. A reference that can't be - pointed at the alias's agent is left with the problem the store ref check reports.""" + """Point each task's reference to an agent or env by an alias an installed bundle gives it, and the configured + default agent when it's one, at what it names, pulled into the plan, unless the store has one of the alias's + id, which the reference keeps naming. A field naming the alias is rewritten to the id. A reference that can't be + pointed at the alias's entity is left with the problem the store ref check reports. Returns the agent aliases + the plan names; ``env_alias_uses`` holds the env aliases.""" if self.installed is None: return {} - table, uses, stored, default = installed_aliases(self.installed()), self.alias_uses, self.alias_stored, _UNREAD - self.alias_table = table + installed = self.installed() + self.alias_table, self.env_alias_table = installed_aliases(installed), installed_aliases(installed, ENV_ALIASES) + tables = {EntityKind.AGENT: self.alias_table, EntityKind.ENV: self.env_alias_table} + uses = {EntityKind.AGENT: self.alias_uses, EntityKind.ENV: self.env_alias_uses} + stored = {EntityKind.AGENT: self.alias_stored, EntityKind.ENV: self.env_alias_stored} + default = _UNREAD for index, entry in enumerate(self.entries): if entry.entry.kind is not BundleKind.TASK: continue references, aliased = [], {} for ref in entry.references: use = None - if ref.kind is EntityKind.AGENT and ref.local is None: - use = self._alias_use(table, ref.id, ref.version, uses, stored) + if ref.kind in tables and ref.local is None: + use = self._alias_use(tables[ref.kind], ref.kind, ref.id, ref.version, uses[ref.kind], + stored[ref.kind]) if isinstance(use, str): self.unpulled[ref.kind, ref.id, ref.version] = use elif use is not None and use.store_version is None: - aliased[ref.id] = use.target_id + aliased[ref.kind, ref.id] = use.target_id ref = self._pulled(replace(ref, id=use.target_id)) references.append(ref) if _leaves_an_agent_unset(entry): if default is _UNREAD: - default = self._alias_use(table, get_config().get_default_a2a_agent_id(), None, uses, stored) + default = self._alias_use(self.alias_table, EntityKind.AGENT, + get_config().get_default_a2a_agent_id(), None, self.alias_uses, + self.alias_stored) if isinstance(default, str): self.problems.append(f"[agents] default_a2a_agent_id: {default}") if isinstance(default, AliasUse) and default.store_version is None: references += self._default_agents(entry, default.target_id) if references != list(entry.references): self._replace(index, replace(entry, config=_renamed(entry.config, aliased), references=tuple(references))) - return uses + return self.alias_uses def _agent_override(self, name: str) -> AgentOverride | None: """The agent ``name`` names, as a task's reference to it would: this bundle's agent, an installed bundle's @@ -327,7 +345,7 @@ def _agent_override(self, name: str) -> AgentOverride | None: else: ref = Reference(EntityKind.AGENT, name, None, None, where, None) use = None if self.alias_table is None else self._alias_use( - self.alias_table, name, None, self.alias_uses, self.alias_stored) + self.alias_table, EntityKind.AGENT, name, None, self.alias_uses, self.alias_stored) if isinstance(use, str): self.problems.append(f"{where}: {use}") return None @@ -358,31 +376,31 @@ def _unstored(self, ref: Reference) -> str | None: return f"agent {ref.id!r} can't be read ({type(e).__name__}: {e})" return None - def _alias_use(self, table: InstalledAliases, agent_id: str, version: int | None, uses: dict[str, AliasUse], - stored: dict[str, int | None]) -> AliasUse | str | None: - """What ``agent_id`` names when it's an installed alias, the store's agent of that id read first (its latest - version, or None, cached in ``stored``), recorded in ``uses``; why it can't be named so; or None when it's no - alias, or names the store's agent as any store reference does: pinned, or with the alias refused.""" - if agent_id.startswith(LOCAL_PREFIX): + def _alias_use(self, table: InstalledAliases, kind: EntityKind, id: str, version: int | None, + uses: dict[str, AliasUse], stored: dict[str, int | None]) -> AliasUse | str | None: + """What ``id``, naming one of ``kind``, names when it's an installed alias, the store's of that id read first + (its latest version, or None, cached in ``stored``), recorded in ``uses``; why it can't be named so; or None + when it's no alias, or names the store's as any store reference does: pinned, or with the alias refused.""" + if id.startswith(LOCAL_PREFIX): return None - refusal, named = table.refusal(agent_id), table.target(agent_id) + refusal, named = table.refusal(id), table.target(id) if refusal is None and named is None: return None - if agent_id not in stored: + if id not in stored: try: - stored[agent_id] = STORE_GETTERS[EntityKind.AGENT](agent_id, None).version + stored[id] = STORE_GETTERS[kind](id, None).version except NotFoundError: - stored[agent_id] = None + stored[id] = None except (ValueError, KeyError, TypeError): # one the store has but can't read, which the store ref check says return None - if stored[agent_id] is not None and (refusal is not None or version is not None): + if stored[id] is not None and (refusal is not None or version is not None): return None if refusal is not None: return refusal if version is not None: - return (f"{agent_id!r} is an alias of {named.agent_id}, which comes with {named.declared_by}, so it has no " + return (f"{id!r} is an alias of {named.target_id}, which comes with {named.declared_by}, so it has no " "fixed version; drop the pin") - return uses.setdefault(agent_id, AliasUse(agent_id, named.agent_id, named.declared_by, stored[agent_id])) + return uses.setdefault(id, AliasUse(id, named.target_id, named.declared_by, stored[id], kind)) def _replace(self, index: int, entry: ResolvedEntry) -> None: """Put ``entry`` in place of the bundle's own entry at ``index``, with what it now references.""" @@ -490,7 +508,7 @@ def plan(self, tasks: Sequence[str], evals: Sequence[str], agent: str | None = N raise BundleError(self.problems) writes = self._ordered(closure) return Plan(self.resolved, writes, tuple(selected_tasks), tuple(selected_evals), store_refs, store_latest, - dict(self.origins), self._published(writes), aliases, override) + dict(self.origins), self._published(writes), aliases, override, dict(self.env_alias_uses)) def check(self) -> None: self._check_writers() @@ -685,7 +703,8 @@ def _store_ref_problem(self, ref: Reference, read: dict) -> str | None: if other is not None: hint = f"; this bundle's {other.kind.store} {other.name!r} has that name" else: - hint = self.alias_table.hint(ref.id) if ref.kind is EntityKind.AGENT and self.alias_table else "" + table = {EntityKind.AGENT: self.alias_table, EntityKind.ENV: self.env_alias_table}.get(ref.kind) + hint = table.hint(ref.id) if table is not None else "" return f"there is no {kind} {ref.id!r}{at} in the store{hint}" if isinstance(found, str): return f"{kind} {ref.id!r} {found}" @@ -1076,8 +1095,9 @@ def _leaves_an_agent_unset(task: ResolvedEntry) -> bool: return False -def _renamed(steps: Any, ids: Mapping[str, str]) -> Any: - """A copy of a task's ``steps`` with each agent field naming an alias in ``ids`` naming that agent's id instead.""" +def _renamed(steps: Any, ids: Mapping[tuple[EntityKind, str], str]) -> Any: + """A copy of a task's ``steps`` with each field naming an alias in ``ids``, by kind, naming what it names + instead.""" if not ids: return steps steps = copy.deepcopy(steps) @@ -1085,8 +1105,8 @@ def _renamed(steps: Any, ids: Mapping[str, str]) -> Any: for step in steps: cls = registry.get(step.get("type")) if isinstance(step.get("type"), str) else None for site in ref_sites(cls.entity_refs or () if cls is not None else (), step): - if site.ref.kind is EntityKind.AGENT and site.ref.role is RefRole.INPUT and site.value in ids: - site.rewrite(ids[site.value]) + if site.ref.role is RefRole.INPUT and (site.ref.kind, site.value) in ids: + site.rewrite(ids[site.ref.kind, site.value]) return steps diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index b05bd02c..d3801146 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -232,17 +232,9 @@ def run_bundle( with namespace_routing(): plan, preflight = _planned(root, tasks, evals, id_root, sandbox, agent) _say_agents(plan, say) - materialization = materialize( - plan, - on_wait=lambda: say("waiting for another agent-env run to finish writing this bundle's ids"), - on_build=lambda write: say(f"{_label(plan, write)}: building with docker, which can take minutes"), - on_write=lambda done: say(_written(plan, done)), - build_here=preflight.built_here, - ) + materialization = _written_with_infra(plan, preflight, say) entries = _to_run(plan) to_run = [(entry, Task.get(entry.id, materialization.version_of("task", entry.id))) for entry in entries] - if preflight.infra: - ensure_default_envs(preflight.infra_kinds, say=say) with Interrupts() as interrupts: overrides = _overrides(sandbox, user_model, judge_model) runs = interrupts.run(_run_all(plan, to_run, model, overrides, _solver_params(plan, materialization), keep, @@ -259,6 +251,40 @@ def run_bundle( raise RunInterrupted(result, interrupts.signum) +def write_bundle( + root: Path | str, + *, + tasks: Sequence[str] = (), + evals: Sequence[str] = (), + sandbox: str | None = None, + on_progress: Callable[[str], None] | None = None, + id_root: str | None = None, +) -> Materialization: + """Write what the tasks and evals selected in ``root`` need and set up the infra envs they deploy on, as + ``run_bundle`` does before it runs them, and run nothing. Raises what ``run_bundle`` raises before a task runs.""" + if sandbox: + build_sandbox_provider(sandbox) + say = _progress(on_progress) + with namespace_routing(): + plan, preflight = _planned(root, tasks, evals, id_root, sandbox, None) + _say_agents(plan, say) + return _written_with_infra(plan, preflight, say) + + +def _written_with_infra(plan: Plan, preflight: Preflight, say: Callable[[str], None]) -> Materialization: + """Write ``plan``, then set up the infra envs its runs deploy on.""" + materialization = materialize( + plan, + on_wait=lambda: say("waiting for another agent-env run to finish writing this bundle's ids"), + on_build=lambda write: say(f"{_label(plan, write)}: building with docker, which can take minutes"), + on_write=lambda done: say(_written(plan, done)), + build_here=preflight.built_here, + ) + if preflight.infra: + ensure_default_envs(preflight.infra_kinds, say=say) + return materialization + + def dry_run_bundle( root: Path | str, *, @@ -348,8 +374,8 @@ def refuse_store_tasks(plan: Plan) -> None: def _say_agents(plan: Plan, say: Callable[[str], None]) -> None: - """Say what each agent alias names, and what the solver deploys in place of the tasks' agents.""" - for use in plan.aliases.values(): + """Say what each agent and env alias names, and what the solver deploys in place of the tasks' agents.""" + for use in (*plan.aliases.values(), *plan.env_aliases.values()): say(_aliased(use)) if plan.agent is not None: runs = _to_run(plan) @@ -485,10 +511,10 @@ def _task_run_command(ref: Reference) -> str: def _aliased(use: AliasUse) -> str: - """Which agent an alias the run names is: the one an installed bundle gives it, or the store's of its id.""" + """Which agent or env an alias the run names is: the one an installed bundle gives it, or the store's of its id.""" if use.store_version is None: return f"{use.alias} → {use.target_id} (the alias {use.declared_by} gives it)" - return (f"{use.alias} → the store's agent {use.alias!r} v{use.store_version}, which shadows the alias " + return (f"{use.alias} → the store's {use.kind.value} {use.alias!r} v{use.store_version}, which shadows the alias " f"{use.declared_by} gives {use.target_id}") diff --git a/src/agent_env/cli/a2a_agent/validate.py b/src/agent_env/cli/a2a_agent/validate.py index f72f776d..f9a0bbae 100644 --- a/src/agent_env/cli/a2a_agent/validate.py +++ b/src/agent_env/cli/a2a_agent/validate.py @@ -5,6 +5,12 @@ import click from agent_env.a2a_agent import A2AAgent +from agent_env.bundle import BundleError, write_bundle +from agent_env.env.bootstrap import InfraError, stores_are_local +from agent_env.env.store import get_env_store +from agent_env.plugins._bundles import ENV_ALIASES, installed_aliases +from agent_env.store.base import NotFoundError +from agent_env.utils.docker.build import DockerBuildError def run_validation(agent: A2AAgent, litellm_api_key: str | None = None) -> dict: @@ -34,7 +40,6 @@ def run_validation(agent: A2AAgent, litellm_api_key: str | None = None) -> dict: def validate(agent_id: str, agent_version: int | None, litellm_api_key: str | None, sandbox: str): """Deploy an A2A agent, validate its agent card and MCP extension.""" from agent_env.providers import build_sandbox_provider, set_agent_sandbox_provider - from agent_env.store.base import NotFoundError if sandbox: try: @@ -50,7 +55,34 @@ def validate(agent_id: str, agent_version: int | None, litellm_api_key: str | No click.echo(click.style(f"Error: {e}", fg="red"), err=True) sys.exit(1) + try: + _write_validation_env(sandbox) + except (BundleError, InfraError, DockerBuildError) as e: + click.echo(click.style(f"Error: {e}", fg="red"), err=True) + sys.exit(1) verifications = run_validation(agent, litellm_api_key=litellm_api_key) if not verifications.get("a2a_agent_mcp", {}).get("passed", False): sys.exit(1) + + +def _write_validation_env(sandbox: str | None) -> None: + """Write the env the validator deploys, ``A2AAgent.VALIDATION_ENV_ID``, when the store has none of that id: the env + an installed bundle gives that alias, with the infra it deploys on, into local stores. Stores that aren't local + need an env of that id of their own, and BundleError says how to put one.""" + env_id = A2AAgent.VALIDATION_ENV_ID + try: + get_env_store().get(env_id) + return + except NotFoundError: + pass + named = installed_aliases(table=ENV_ALIASES).target(env_id) + if named is None: # deploying it says why it can't be found + return + if not stores_are_local(): + dockerfile = named.bundle.root / "envs" / named.target_id.rsplit("/", 1)[1] / "Dockerfile" + raise BundleError([f"the store has no env {env_id!r}, the one validate deploys, and validate writes " + f"{named.declared_by}'s only into local stores; put it with `agent-env env mcp-server put " + f"--id {env_id} --dockerfile {dockerfile}`"]) + click.echo(f"{env_id} → {named.target_id} (the alias {named.declared_by} gives it): writing it") + write_bundle(named.bundle.root, id_root=named.bundle.id_root, sandbox=sandbox, on_progress=click.echo) diff --git a/src/agent_env/cli/plugin.py b/src/agent_env/cli/plugin.py index 4c1ebe00..591a950c 100644 --- a/src/agent_env/cli/plugin.py +++ b/src/agent_env/cli/plugin.py @@ -35,7 +35,7 @@ ) from agent_env.plugins._cli import CLI_PLUGINS_GROUP, CLI_ROOT_OPTIONS_GROUP, cli_contributions, cli_discovery_errors from agent_env.plugins._discovery import discovery_error -from agent_env.plugins._bundles import InstalledAliases, installed_aliases +from agent_env.plugins._bundles import ENV_ALIASES, InstalledAliases, installed_aliases from agent_env.plugins._report import ( ALIAS_REFUSED, ENTRY_POINTS_UNREADABLE, @@ -154,17 +154,17 @@ def _bundle_contributions(*, load: bool) -> tuple[dict[tuple[str, str], list[Con except Exception as e: return {}, {BUNDLES: Diagnostic(ENTRY_POINTS_UNREADABLE, discovery_error(e))} found: dict[tuple[str, str], list[Contribution]] = {} - aliases = installed_aliases(bundles) + aliases = (installed_aliases(bundles), installed_aliases(bundles, ENV_ALIASES)) for bundle in bundles: found.setdefault((bundle.package, bundle.version), []).append( _bundle_contribution(bundle, bundles, aliases, load=load)) return found, {} -def _bundle_contribution(bundle: InstalledBundle, bundles: tuple[InstalledBundle, ...], aliases: InstalledAliases, *, - load: bool) -> Contribution: +def _bundle_contribution(bundle: InstalledBundle, bundles: tuple[InstalledBundle, ...], + aliases: tuple[InstalledAliases, ...], *, load: bool) -> Contribution: """Whether ``bundle`` runs, by the checks ``agent-env run`` makes before it reads a store, and whether it gets each - alias it gives an agent. Without loading plugins, only that the folder parses.""" + alias it gives an agent or env. Without loading plugins, only that the folder parses.""" namesakes = tuple(Claimant(other.package, other.version, other.value) for other in bundles if other.name == bundle.name and other is not bundle) if bundle.problem: @@ -176,7 +176,7 @@ def _bundle_contribution(bundle: InstalledBundle, bundles: tuple[InstalledBundle except BundleError as e: return Contribution(BUNDLES, bundle.name, bundle.value, "failed", code=INVALID_PLUGIN, reason=f"it isn't a valid bundle: {e.summary}") - refused = aliases.refused_claims.get(bundle.qualified, ()) + refused = tuple(why for table in aliases for why in table.refused_claims.get(bundle.qualified, ())) if run_name(bundle, bundles) != bundle.name: qualified = f"another package also installs {bundle.name!r}, so run this one as {bundle.qualified!r}" return Contribution(BUNDLES, bundle.name, bundle.value, "active", code=QUALIFIED_ONLY, diff --git a/src/agent_env/cli/run.py b/src/agent_env/cli/run.py index d0138616..18a5c62d 100644 --- a/src/agent_env/cli/run.py +++ b/src/agent_env/cli/run.py @@ -13,7 +13,7 @@ from agent_env.bundle import (BundleError, BundleKind, BundleRun, DryRun, Outcome, RunInterrupted, TaskRun, dry_run_bundle, run_bundle) from agent_env.bundle.installed import InstalledBundle, checked, find_bundle, installed_bundles, run_name -from agent_env.plugins._bundles import installed_aliases +from agent_env.plugins._bundles import ENV_ALIASES, installed_aliases from agent_env.plugins._discovery import discovery_error from agent_env.providers.sandbox_providers.local_sandbox import LocalSandbox from agent_env.task.interrupts import Interrupts @@ -157,12 +157,12 @@ def echo(name: str, package: str, contents: str, description: str) -> None: click.echo(f"{name:<{widths[0]}} {package:<{widths[1]}} {contents:<{widths[2]}} {description}".rstrip()) echo(*header) - aliases = installed_aliases(bundles) + aliases = (installed_aliases(bundles), installed_aliases(bundles, ENV_ALIASES)) for bundle, row in zip(bundles, rows): echo(*row) if bundle.root is not None: click.echo(f"{'':<{widths[0]}} {bundle.root}") - if named := aliases.of(bundle): + if named := [alias for table in aliases for alias in table.of(bundle)]: click.echo(f"{'':<{widths[0]}} aliases: {', '.join(named)}") click.echo("\nRun one with agent-env run NAME. To change one, copy its folder and run the copy.") diff --git a/src/agent_env/env/env.py b/src/agent_env/env/env.py index bfc38276..a429af91 100644 --- a/src/agent_env/env/env.py +++ b/src/agent_env/env/env.py @@ -1,5 +1,6 @@ from __future__ import annotations +import copy import logging import posixpath import shlex @@ -10,6 +11,9 @@ from agentenv_protocol import client as protocol_v1 from agent_env.entity.refs import EntityRef +from agent_env.plugins._bundles import ENV_ALIASES, installed_aliases +from agent_env.store.base import NotFoundError +from agent_env.store.routing import namespace_routing_enabled from agent_env.utils.paths import validate_relative_filename from .gateway import GatewayMode @@ -60,6 +64,15 @@ def __init__(self, what: str, env_id: str) -> None: super().__init__(f"{what} needs a sandbox; env '{env_id}' runs outside agent-env's sandboxes") +def _holds(env_id: str) -> bool: + """Whether the store has an env of ``env_id``, at any version.""" + try: + get_env_store().get(env_id) + except NotFoundError: + return False + return True + + @dataclass(kw_only=True) class DeployedEnv: """A deployed env: the env, its card, and the registry's bookkeeping. Each topology's subclass adds what it created.""" @@ -227,8 +240,37 @@ def __init__(self, id: str, version: Optional[int], metadata: Optional[dict[str, @classmethod def get(cls, id: str, version: Optional[int] = None) -> "Env": - from .store import get_env_store - return get_env_store().get(id, version) + """The env ``id`` names: the store's env of that id, else the env an installed bundle gives the alias ``id``, + which the store holds once a run of a bundle has used it. The alias's env answers to the alias's id, as the + store's env of that id would, so a run's records name the env its task names; ``version`` is its env's. An alias + names its env only where namespace routing is on, as in the CLI. Raises NotFoundError.""" + store = get_env_store() + try: + return store.get(id, version) + except NotFoundError: + if not isinstance(id, str) or id.startswith("@") or "/" in id: # neither an alias nor close to one + raise + aliases = installed_aliases(table=ENV_ALIASES) + named, refusal = aliases.target(id), aliases.refusal(id) + if named is None and refusal is None: + if version is None and (hint := aliases.hint(id)): + raise NotFoundError(f"there is no env {id!r} in the store{hint}") from None + raise + if version is not None and _holds(id): + raise + if named is None: + raise NotFoundError(refusal) from None + if not namespace_routing_enabled(): + raise NotFoundError(f"{id!r} is an alias of {named.target_id}, which comes with {named.declared_by}, and " + f"an alias names its env only in the CLI; put an env of the id {id!r} in the store") + try: + env = store.get(named.target_id, version) + except NotFoundError: + raise NotFoundError(f"{id!r} is an alias of {named.target_id}, which comes with {named.declared_by} and " + "isn't in the store yet: an `agent-env run` of a bundle that uses it writes it") from None + env = copy.copy(env) + env.id = id + return env @classmethod async def from_instance_id(cls, instance_id: str) -> "Env": diff --git a/src/agent_env/examples/mail-mcp/README.md b/src/agent_env/examples/mail-mcp/README.md new file mode 100644 index 00000000..fb43ad3c --- /dev/null +++ b/src/agent_env/examples/mail-mcp/README.md @@ -0,0 +1,10 @@ +An MCP server with an inbox to search, deployed through the gateway: the env the A2A agent validator gives agents. + +`agent-env a2a-agent validate` deploys it, by its alias, beside the agent it checks. It needs Docker. `envs/mail/` is +the server: a `Dockerfile` and a `server.py` whose one tool, `search_emails`, returns the emails holding every word of +its query from a fixed inbox of three, named `mail` by its environment card. `aliases.toml` gives it the alias +`mail-mcp`, so any task can name it `mail-mcp`. `tasks/smoke.json` deploys it and checks, with +`artifacts/check_mail/verify.py`, that a search for "invoice" finds the inbox's one invoice. The first run also builds +the gateway and service database it deploys on, which takes minutes. + +Run it with `agent-env run mail-mcp`, or copy this folder and run the copy with `agent-env run ./my-mail`. diff --git a/src/agent_env/examples/mail-mcp/aliases.toml b/src/agent_env/examples/mail-mcp/aliases.toml new file mode 100644 index 00000000..9fddc0ef --- /dev/null +++ b/src/agent_env/examples/mail-mcp/aliases.toml @@ -0,0 +1,3 @@ +# Ids a task can name this bundle's envs by; a store's env of the id wins over an alias. +[envs] +mail = ["mail-mcp"] diff --git a/src/agent_env/examples/mail-mcp/artifacts/check_mail/verify.py b/src/agent_env/examples/mail-mcp/artifacts/check_mail/verify.py new file mode 100644 index 00000000..3d360af1 --- /dev/null +++ b/src/agent_env/examples/mail-mcp/artifacts/check_mail/verify.py @@ -0,0 +1,15 @@ +"""Calls the search_emails tool through the env's MCP endpoint and checks it finds the inbox's one invoice.""" +import json + +from mcp import ClientSession +from mcp.client.streamable_http import streamable_http_client + + +async def verify(mcp_url): + async with streamable_http_client(mcp_url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + result = await session.call_tool("search_emails", {"query": "invoice"}) + text = "".join(getattr(part, "text", "") for part in result.content) + found = [email["id"] for email in json.loads(text)] if not result.isError else [] + return [{"id": "finds-the-invoice", "result": found == ["m-1"], "description": f"search_emails found {found}"}] diff --git a/src/agent_env/examples/mail-mcp/envs/mail/Dockerfile b/src/agent_env/examples/mail-mcp/envs/mail/Dockerfile new file mode 100644 index 00000000..f7aeb5dd --- /dev/null +++ b/src/agent_env/examples/mail-mcp/envs/mail/Dockerfile @@ -0,0 +1,8 @@ +FROM public.ecr.aws/docker/library/python:3.11-slim@sha256:c20888b6acdd1e63e1c433a185bf3ad162c0288fe484616ce062e0d28add2900 +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY server.py . +RUN useradd --create-home --uid 10001 appuser +USER appuser +CMD ["python", "server.py"] diff --git a/src/agent_env/examples/mail-mcp/envs/mail/requirements.txt b/src/agent_env/examples/mail-mcp/envs/mail/requirements.txt new file mode 100644 index 00000000..2f549484 --- /dev/null +++ b/src/agent_env/examples/mail-mcp/envs/mail/requirements.txt @@ -0,0 +1,3 @@ +# A release pins agentenv-framework-protocol to the version it was released with. +agentenv-framework-protocol +mcp>=1.25,<2 diff --git a/src/agent_env/examples/mail-mcp/envs/mail/server.py b/src/agent_env/examples/mail-mcp/envs/mail/server.py new file mode 100644 index 00000000..298dc0cb --- /dev/null +++ b/src/agent_env/examples/mail-mcp/envs/mail/server.py @@ -0,0 +1,32 @@ +"""The mail env: an MCP server with one tool, which searches a small, fixed inbox.""" +import json +from typing import Annotated + +from pydantic import Field + +from agentenv_protocol import AgentEnvEnvironment, environment_card, tool + +INBOX = ( + {"id": "m-1", "from": "billing@acme-supplies.example", "subject": "Invoice INV-1042 for October", + "body": "Invoice INV-1042 for $2,740.00 is due on November 15."}, + {"id": "m-2", "from": "dana@example.com", "subject": "Team lunch on Friday", + "body": "Pizza at noon in the kitchen."}, + {"id": "m-3", "from": "it@example.com", "subject": "Your password expires soon", + "body": "Choose a new password within 7 days."}, +) + + +@environment_card(name="mail") +class MailEnv(AgentEnvEnvironment): + @tool(name="search_emails") + def search_emails( + self, + query: Annotated[str, Field(description="Words each email returned holds, in its sender, subject or body.")], + ) -> str: + """Search the inbox: the emails holding every word of the query, ignoring case, as a JSON list.""" + words = query.lower().split() + return json.dumps([email for email in INBOX if all(word in " ".join(email.values()).lower() for word in words)]) + + +if __name__ == "__main__": + MailEnv().serve() diff --git a/src/agent_env/examples/mail-mcp/tasks/smoke.json b/src/agent_env/examples/mail-mcp/tasks/smoke.json new file mode 100644 index 00000000..3eda8b76 --- /dev/null +++ b/src/agent_env/examples/mail-mcp/tasks/smoke.json @@ -0,0 +1,5 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "mail"}, + {"id": "finds-the-invoice", "type": "env_outcome_verifier", "env_id": "mail", "file_artifact_id": "check_mail", + "verifier_id": "finds-the-invoice"} +] diff --git a/src/agent_env/plugins/_bundles.py b/src/agent_env/plugins/_bundles.py index 27c0cfdd..0c163b7e 100644 --- a/src/agent_env/plugins/_bundles.py +++ b/src/agent_env/plugins/_bundles.py @@ -4,8 +4,8 @@ found without importing any package code: only the top-level package is located, and the rest of the path is walked as folders. An installed bundle's ids are rooted at its distribution, ``@local//``. -A bundle can give its agents short aliases in an ``aliases.toml`` at its root, which a reference names an agent by -when no agent of that id is in the store (``installed_aliases``). +A bundle can give its agents and envs short aliases in an ``aliases.toml`` at its root, which a reference names one by +when no agent or env of that id is in the store (``installed_aliases``). It imports nothing from ``agent_env.bundle``, so a task step can read what's installed without importing the bundle machinery, whose run imports the task steps. @@ -35,6 +35,9 @@ BUNDLES = "agent_env.bundles" CORE = "agentenv-framework" ALIASES_FILE = "aliases.toml" +# The tables an aliases.toml takes, each giving aliases to one kind of the bundle's entities, by folder name. +AGENT_ALIASES, ENV_ALIASES = "agents", "envs" +_NOUNS = {AGENT_ALIASES: "agent", ENV_ALIASES: "env"} # A plain id, never a bundle's namespaced one, which holds '@' and '/'. _ALIAS = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?") _ALIAS_RULE = "lowercase letters, digits and '-', at most 64 characters, starting and ending with a letter or digit" @@ -141,10 +144,10 @@ def _spelled(location: Path, parts: tuple[str, ...]) -> Path | None: @dataclass(frozen=True) class BundleAlias: - """An alias an installed bundle gives one of its agents.""" + """An alias an installed bundle gives one of its agents or envs.""" alias: str - agent_id: str # ``/`` + target_id: str # ``/`` bundle: InstalledBundle @property @@ -154,12 +157,13 @@ def declared_by(self) -> str: @dataclass(frozen=True) class InstalledAliases: - """The aliases installed bundles give their agents: each alias that names an agent, each refused to every bundle - that declares it, with why, and each bundle's refused claims, with why.""" + """The aliases installed bundles give their entities of one kind, an ``aliases.toml`` table: each alias that names + one, each refused to every bundle that declares it, with why, and each bundle's refused claims, with why.""" aliases: Mapping[str, BundleAlias] refused: Mapping[str, str] refused_claims: Mapping[str, tuple[str, ...]] # by the bundle's qualified name + table: str = AGENT_ALIASES def target(self, alias: str) -> BundleAlias | None: return self.aliases.get(alias) @@ -168,25 +172,25 @@ def refusal(self, alias: str) -> str | None: return self.refused.get(alias) def of(self, bundle: InstalledBundle) -> tuple[str, ...]: - """The aliases that name ``bundle``'s agents, in name order.""" + """The aliases that name ``bundle``'s entities, in name order.""" return tuple(sorted(alias for alias, named in self.aliases.items() if named.bundle == bundle)) - def naming(self, agent_id: str) -> tuple[str, ...]: - """The aliases that name the agent ``agent_id``, in name order.""" - return tuple(sorted(alias for alias, named in self.aliases.items() if named.agent_id == agent_id)) + def naming(self, target_id: str) -> tuple[str, ...]: + """The aliases that name the entity ``target_id``, in name order.""" + return tuple(sorted(alias for alias, named in self.aliases.items() if named.target_id == target_id)) def hint(self, name: str) -> str: - """For ``name``, which names no agent: the aliases close to it, or ''.""" + """For ``name``, which names nothing of this kind: the aliases close to it, or ''.""" close = difflib.get_close_matches(name, sorted(self.aliases), n=3) if not close: return "" - return (f"; did you mean the alias {' or '.join(repr(alias) for alias in close)}? agent-env a2a-agent list " - "lists the agents a task can name") + listed = " agent-env a2a-agent list lists the agents a task can name" if self.table == AGENT_ALIASES else "" + return f"; did you mean the alias {' or '.join(repr(alias) for alias in close)}?{listed}" -def read_aliases(path: Path) -> tuple[dict[str, tuple[str, ...]], list[str]]: - """The aliases a bundle's ``aliases.toml`` at ``path`` gives its agents, by agent folder name, and its problems; none - of either when there's no such file, and no aliases when it has a problem.""" +def read_aliases(path: Path) -> tuple[dict[str, dict[str, tuple[str, ...]]], list[str]]: + """The aliases a bundle's ``aliases.toml`` at ``path`` gives its agents and envs, by table and then folder name, and + its problems; none of either when there's no such file, and no aliases when it has a problem.""" if not os.path.lexists(path): return {}, [] if not path.is_file(): @@ -197,40 +201,43 @@ def read_aliases(path: Path) -> tuple[dict[str, tuple[str, ...]], list[str]]: return {}, [f"can't be read: {os_reason(e)}"] except (UnicodeDecodeError, tomllib.TOMLDecodeError) as e: return {}, [f"isn't valid TOML: {e}"] - problems = [f"[{key}] isn't a table it takes; it takes [agents]" for key in data if key != "agents"] - agents = data.get("agents", {}) - if not isinstance(agents, dict): - return {}, [*problems, "[agents] must be a table of agent folder names to lists of aliases"] - found: dict[str, tuple[str, ...]] = {} - named: dict[str, str] = {} - for name, aliases in agents.items(): - if not (isinstance(aliases, list) and aliases and all(isinstance(alias, str) for alias in aliases)): - problems.append(f"agents.{name} must be a list of aliases, not {aliases!r}") + problems = [f"[{key}] isn't a table it takes; it takes [agents] and [envs]" for key in data if key not in _NOUNS] + found: dict[str, dict[str, tuple[str, ...]]] = {} + for table, noun in _NOUNS.items(): + entries = data.get(table, {}) + if not isinstance(entries, dict): + problems.append(f"[{table}] must be a table of {noun} folder names to lists of aliases") continue - for alias in aliases: - if not _ALIAS.fullmatch(alias): - problems.append(f"agents.{name}: {alias!r} isn't an alias: {_ALIAS_RULE}") - elif alias in named: - where = "twice" if named[alias] == name else f"for agents.{named[alias]} too" - problems.append(f"agents.{name}: {alias!r} is listed {where}; an alias names one agent") - else: - named[alias] = name - found[name] = tuple(aliases) + named: dict[str, str] = {} + for name, aliases in entries.items(): + if not (isinstance(aliases, list) and aliases and all(isinstance(alias, str) for alias in aliases)): + problems.append(f"{table}.{name} must be a list of aliases, not {aliases!r}") + continue + for alias in aliases: + if not _ALIAS.fullmatch(alias): + problems.append(f"{table}.{name}: {alias!r} isn't an alias: {_ALIAS_RULE}") + elif alias in named: + where = "twice" if named[alias] == name else f"for {table}.{named[alias]} too" + problems.append(f"{table}.{name}: {alias!r} is listed {where}; an alias names one {noun}") + else: + named[alias] = name + found.setdefault(table, {})[name] = tuple(aliases) return ({}, problems) if problems else (found, []) -def installed_aliases(bundles: Sequence[InstalledBundle] | None = None) -> InstalledAliases: - """The aliases the installed bundles give their agents. agent-env's own bundles' aliases are theirs: another bundle - that declares one is refused it. An alias two other bundles both declare names neither, so a reference to it is - refused rather than given one of them. A bundle whose ``aliases.toml`` has a problem gives none, which checking the - bundle reports.""" +def installed_aliases(bundles: Sequence[InstalledBundle] | None = None, table: str = AGENT_ALIASES) -> InstalledAliases: + """The aliases the installed bundles give their entities of ``table``'s kind. agent-env's own bundles' aliases are + theirs: another bundle that declares one is refused it. An alias two other bundles both declare names neither, so a + reference to it is refused rather than given one of them. A bundle whose ``aliases.toml`` has a problem gives none, + which checking the bundle reports.""" bundles = installed_bundles() if bundles is None else bundles + noun = _NOUNS[table] claims_by_alias: dict[str, list[BundleAlias]] = {} for bundle in bundles: if bundle.root is None or bundle.problem: continue declared, _ = read_aliases(bundle.root / ALIASES_FILE) - for name, aliases in declared.items(): + for name, aliases in declared.get(table, {}).items(): for alias in aliases: claims_by_alias.setdefault(alias, []).append(BundleAlias(alias, f"{bundle.id_root}/{name}", bundle)) aliases: dict[str, BundleAlias] = {} @@ -243,15 +250,15 @@ def installed_aliases(bundles: Sequence[InstalledBundle] | None = None) -> Insta for claim in claimants: if claim.bundle.dist != core[0].bundle.dist: refused_claims.setdefault(claim.bundle.qualified, []).append( - f"{alias!r} is agent-env's own alias, of {core[0].agent_id}, so this bundle can't give it") + f"{alias!r} is agent-env's own alias, of {core[0].target_id}, so this bundle can't give it") elif len(claimants) == 1: aliases[alias] = claimants[0] else: declared_by = " and ".join(claim.declared_by for claim in claimants) - refused[alias] = (f"{alias!r} is an alias {declared_by} each give an agent, so it names none of them; " - "name the agent by its id") + refused[alias] = (f"{alias!r} is an alias {declared_by} each give an {noun}, so it names none of " + f"them; name the {noun} by its id") for claim in claimants: refused_claims.setdefault(claim.bundle.qualified, []).append( f"{alias!r} is another installed bundle's alias too, so neither gets it") - return InstalledAliases(aliases, refused, {name: tuple(why) for name, why in refused_claims.items()}) + return InstalledAliases(aliases, refused, {name: tuple(why) for name, why in refused_claims.items()}, table) diff --git a/tst/integration/cli/run_bundle_built_envs_test.py b/tst/integration/cli/run_bundle_built_envs_test.py index 8299dbaa..9163312b 100644 --- a/tst/integration/cli/run_bundle_built_envs_test.py +++ b/tst/integration/cli/run_bundle_built_envs_test.py @@ -564,6 +564,24 @@ def test_the_shipped_echo_mcp_example_runs_by_name_and_a_rerun_writes_nothing(st assert "tasks/smoke.json v1: passed (echoes: 1)" in second.output, second.output + +def test_the_shipped_mail_mcp_example_runs_and_a_task_outside_a_bundle_deploys_its_env_by_its_alias(state): + """The alias is how the A2A agent validator's task, which no bundle holds, names the env it deploys.""" + first = _run("mail-mcp") + + assert first.exit_code == 0, first.output + assert "tasks/smoke.json v1: passed (finds-the-invoice: 1)" in first.output, first.output + + (state / "by-alias.json").write_text(json.dumps([{"id": "env", "type": "deploy_env", "env_id": "mail-mcp"}])) + created = CliRunner().invoke(cli, ["task", "create", "--id", "by-alias", str(state / "by-alias.json")]) + ran = CliRunner().invoke(cli, ["task", "run", "--id", "by-alias", "--output-dir", str(state / "out")]) + + assert created.exit_code == 0, created.output + assert ran.exit_code == 0, ran.output + [written] = (state / "out").iterdir() + [deployed] = json.loads(written.read_text())["deployed_envs"] + assert deployed["env_id"] == "mail-mcp" and deployed["gateway_url"], deployed + ECHO = REPO / "src" / "agent_env" / "examples" / "echo-mcp" diff --git a/tst/unit/bundle/aliases_test.py b/tst/unit/bundle/aliases_test.py index 8de828da..6547b337 100644 --- a/tst/unit/bundle/aliases_test.py +++ b/tst/unit/bundle/aliases_test.py @@ -5,7 +5,7 @@ import pytest from agent_env.bundle import BundleError, parse_bundle -from agent_env.plugins._bundles import ALIASES_FILE, InstalledBundle, installed_aliases, read_aliases +from agent_env.plugins._bundles import ALIASES_FILE, ENV_ALIASES, InstalledBundle, installed_aliases, read_aliases from tst.unit.bundle._support import layout @@ -22,9 +22,11 @@ def _read(tmp_path, text): return read_aliases(tmp_path / ALIASES_FILE) -def test_an_aliases_file_gives_each_agent_its_aliases(tmp_path): - assert _read(tmp_path, '[agents]\nclaude-code = ["claude-code", "claude"]\npi = ["pi"]\n') == ( - {"claude-code": ("claude-code", "claude"), "pi": ("pi",)}, []) +def test_an_aliases_file_gives_each_agent_and_env_its_aliases(tmp_path): + text = '[agents]\nclaude-code = ["claude-code", "claude"]\npi = ["pi"]\n[envs]\nmail = ["mail-mcp"]\n' + + assert _read(tmp_path, text) == ( + {"agents": {"claude-code": ("claude-code", "claude"), "pi": ("pi",)}, "envs": {"mail": ("mail-mcp",)}}, []) def test_a_bundle_without_an_aliases_file_gives_none(tmp_path): @@ -42,11 +44,12 @@ def test_a_bundle_without_an_aliases_file_gives_none(tmp_path): ('[agents]\nx = []\n', "agents.x must be a list of aliases, not []"), ('[agents]\nx = ["a"]\ny = ["a"]\n', "agents.y: 'a' is listed for agents.x too; an alias names one agent"), ('[agents]\nx = ["a", "a"]\n', "agents.x: 'a' is listed twice"), - ('[envs]\nx = ["a"]\n', "[envs] isn't a table it takes; it takes [agents]"), + ('[skills]\nx = ["a"]\n', "[skills] isn't a table it takes; it takes [agents] and [envs]"), + ('[envs]\nx = ["a", "a"]\n', "envs.x: 'a' is listed twice; an alias names one env"), ('agents = 3\n', "[agents] must be a table"), ('[agents\n', "isn't valid TOML"), ], ids=["case", "namespaced", "slash", "edge", "long", "final-newline", "not-a-list", "empty", "two-agents", "twice", "other-table", - "not-a-table", "toml"]) + "env-twice", "not-a-table", "toml"]) def test_an_aliases_file_with_a_problem_gives_no_aliases_and_says_why(tmp_path, text, problem): aliases, problems = _read(tmp_path, text) @@ -69,7 +72,7 @@ def test_an_installed_bundles_alias_names_its_agent_under_the_bundles_id_root(tm table = installed_aliases((agents,)) named = table.target("claude-code") - assert (named.agent_id, named.bundle, named.declared_by) == ( + assert (named.target_id, named.bundle, named.declared_by) == ( "@local/agentenv-framework/agents/claude-code", agents, "agentenv-framework's agents bundle") assert table.of(agents) == ("claude-code",) and table.target("pi") is None and table.refusal("claude-code") is None @@ -86,6 +89,17 @@ def test_agent_envs_own_aliases_are_refused_to_any_other_bundle(tmp_path): "can't give it",)} +def test_an_installed_bundle_gives_its_envs_aliases_of_their_own(tmp_path): + """An env alias and an agent alias are kept apart, each where a reference to its kind looks.""" + mail = _bundle(tmp_path, "mail-mcp", aliases='[envs]\nmail = ["mail-mcp"]\n[agents]\nhelper = ["mail-mcp"]\n') + + envs, agents = installed_aliases((mail,), ENV_ALIASES), installed_aliases((mail,)) + + assert envs.target("mail-mcp").target_id == "@local/agentenv-framework/mail-mcp/mail" + assert agents.target("mail-mcp").target_id == "@local/agentenv-framework/mail-mcp/helper" + assert envs.hint("mail_mcp") == "; did you mean the alias 'mail-mcp'?" + + def test_a_name_close_to_an_alias_is_pointed_at_it(tmp_path): table = installed_aliases((_bundle(tmp_path, "agents", aliases='[agents]\nclaude-code = ["claude-code"]\n'),)) @@ -130,14 +144,16 @@ def test_a_bundle_gives_its_agents_the_aliases_its_aliases_file_lists(tmp_path): bundle = parse_bundle(root, id_root="@local/acme-kit/kit") - assert bundle.aliases == {"solver": ("solver", "fixer")} and ALIASES_FILE not in map(str, bundle.ignored) + assert bundle.aliases == {"agents": {"solver": ("solver", "fixer")}} + assert ALIASES_FILE not in map(str, bundle.ignored) @pytest.mark.parametrize("aliases, problem", [ ('[agents]\nnobody = ["x"]\n', "aliases.toml: agents.nobody: this bundle has no agent 'nobody'"), ('[agents]\nsolver = ["helper"]\n', "aliases.toml: agents.solver: 'helper' is the agent 'helper''s name"), ('[agents]\nsolver = ["Solver"]\n', "aliases.toml: agents.solver: 'Solver' isn't an alias"), -], ids=["no-such-agent", "another-agents-name", "syntax"]) + ('[envs]\nsolver = ["x"]\n', "aliases.toml: envs.solver: this bundle has no env 'solver'"), +], ids=["no-such-agent", "another-agents-name", "syntax", "an-agent-as-an-env"]) def test_an_aliases_file_that_doesnt_fit_the_bundles_agents_is_refused(tmp_path, aliases, problem): root = layout(tmp_path / "kit", {**_AGENTS, ALIASES_FILE: aliases}) @@ -146,6 +162,13 @@ def test_an_aliases_file_that_doesnt_fit_the_bundles_agents_is_refused(tmp_path, assert found.startswith(problem) +def test_a_bundle_gives_its_envs_the_aliases_its_aliases_file_lists(tmp_path): + root = layout(tmp_path / "kit", {**_AGENTS, "envs/mail/Dockerfile": "FROM scratch\n", + ALIASES_FILE: '[envs]\nmail = ["mail-mcp"]\n'}) + + assert parse_bundle(root, id_root="@local/acme-kit/kit").aliases == {"envs": {"mail": ("mail-mcp",)}} + + def test_an_alias_names_an_agent_by_its_folder_so_the_agent_keeps_its_folders_id(tmp_path): root = layout(tmp_path / "kit", {**_AGENTS, "agents/solver/agent.toml": 'id = "@local/acme-kit/elsewhere"\n', ALIASES_FILE: '[agents]\nsolver = ["solver"]\n'}) diff --git a/tst/unit/bundle/installed_entities_test.py b/tst/unit/bundle/installed_entities_test.py index a628a524..8eeb3c26 100644 --- a/tst/unit/bundle/installed_entities_test.py +++ b/tst/unit/bundle/installed_entities_test.py @@ -21,6 +21,8 @@ from agent_env.bundle.plan import AliasUse, check_bundle, plan_bundle from agent_env.bundle.resolve import resolve_bundle from agent_env.plugins._bundles import ALIASES_FILE +from agent_env.entity.refs import EntityKind +from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.store.routing import namespace_routing from tst.unit.bundle._support import RefusingStore, layout, local_builds @@ -362,6 +364,38 @@ def test_an_agent_named_by_an_installed_alias_is_pulled_in_and_the_task_names_it assert plan.aliases["solver-x"].agent_id == SOLVER and plan.store_refs == () +ENV_ALIASED = {"envs/crm/Dockerfile": "FROM scratch\n", "envs/crm/env.toml": 'environment_name = "crm"\n', + ALIASES_FILE: '[envs]\ncrm = ["crm-x"]\n'} +CRM = f"{KIT}/crm" +CRM_STEPS = [{"id": "env", "type": "deploy_env", "env_id": "crm-x"}, + {"id": "agent", "type": "deploy_agent", "a2a_agent_id": SOLVER, "env_ids": ["crm-x"], + "depends_on": ["env"]}] + + +def test_an_env_named_by_an_installed_alias_is_pulled_in_and_the_task_names_it_by_id(bundle_dir, tmp_path): + kit = _kit(tmp_path, ENV_ALIASED) + + plan = _plan(_task(bundle_dir, CRM_STEPS), kit) + + assert (BundleKind.ENV, CRM) in _writes(plan) + assert _step(plan, "env")["env_id"] == CRM and _step(plan, "agent")["env_ids"] == [CRM] + assert plan.env_aliases == {"crm-x": AliasUse("crm-x", CRM, "demo-dist's kit bundle", kind=EntityKind.ENV)} + assert plan.aliases == {} and plan.store_refs == () + + +def test_the_stores_env_of_an_alias_id_shadows_the_alias(bundle_dir, tmp_path): + kit = _kit(tmp_path, ENV_ALIASED) + with namespace_routing(): + image = get_artifact_store().put_document(DockerImageArtifact( + id="crm-x-image", description="d", image_name="ghcr.io/acme/crm:1", tar_gz_s3_url="s3://b/x.tar.gz")) + MCPServerEnv.put(id="crm-x", docker_image_artifact=image, environment_name="crm") + + plan = _plan(_task(bundle_dir, CRM_STEPS), kit) + + assert (BundleKind.ENV, CRM) not in _writes(plan) and _step(plan, "env")["env_id"] == "crm-x" + assert plan.env_aliases["crm-x"].store_version == 1 and [ref.id for ref in plan.store_refs] == ["crm-x"] + + def test_the_stores_agent_of_an_alias_id_shadows_the_alias(bundle_dir, tmp_path): """Someone's own agent of that id is the one they mean.""" kit = _kit(tmp_path, ALIASED) diff --git a/tst/unit/bundle/installed_test.py b/tst/unit/bundle/installed_test.py index cc2ceca0..699a8f8b 100644 --- a/tst/unit/bundle/installed_test.py +++ b/tst/unit/bundle/installed_test.py @@ -127,7 +127,8 @@ def test_the_plugin_report_counts_the_bundles_agent_env_ships(monkeypatch, quiet assert result.exit_code == 0, result.output (core,) = json.loads(result.stdout)["plugins"] assert [(c["group"], c["name"], c["status"]) for c in core["contributions"]] == [ - (BUNDLES, "agents", "active"), (BUNDLES, "echo-mcp", "active"), (BUNDLES, "hello", "active")] + (BUNDLES, "agents", "active"), (BUNDLES, "echo-mcp", "active"), (BUNDLES, "hello", "active"), + (BUNDLES, "mail-mcp", "active")] assert "\n agent-env itself\n" in shown and "requires:" not in shown diff --git a/tst/unit/bundle/preflight_test.py b/tst/unit/bundle/preflight_test.py index 9053bd7a..818b3ead 100644 --- a/tst/unit/bundle/preflight_test.py +++ b/tst/unit/bundle/preflight_test.py @@ -17,7 +17,7 @@ from agent_env.artifact.artifacts import docker_image as docker_image_module from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.store import get_artifact_store -from agent_env.bundle import BundleError, dry_run_bundle, run_bundle +from agent_env.bundle import BundleError, BundleKind, dry_run_bundle, run_bundle, write_bundle from agent_env.cli import cli from agent_env.config import get_config from agent_env.config.runtime import Config @@ -992,6 +992,25 @@ async def deploy(self, context): assert ("deploy", "crm") in events + +def test_write_bundle_writes_and_sets_up_the_infra_its_tasks_deploy_on_and_runs_nothing(bundle_dir, monkeypatch): + _env("crm") + _task(bundle_dir, [{"id": "env", "type": "deploy_env", "env_id": "crm"}]) + events = [] + monkeypatch.setattr("agent_env.bundle.run.ensure_default_envs", + lambda kinds, say: events.append(("bootstrap", sorted(kinds)))) + + async def deploy(self, context): + events.append(("deploy", self.env_id)) + return context + + monkeypatch.setattr(DeployEnvTaskStep, "execute", deploy) + + materialization = write_bundle(bundle_dir, on_progress=lambda line: events.append(("say", line))) + + assert events == [("say", "tasks/t.json: v1 (new)"), ("bootstrap", ["gateway", "service-db"])] + assert [done.write.kind for done in materialization.writes] == [BundleKind.TASK] + # The walk resolves each deploy's provider as its step does diff --git a/tst/unit/bundle/shipped_bundles_test.py b/tst/unit/bundle/shipped_bundles_test.py index 64ab0045..77c8b9e9 100644 --- a/tst/unit/bundle/shipped_bundles_test.py +++ b/tst/unit/bundle/shipped_bundles_test.py @@ -8,12 +8,13 @@ import pytest +from agent_env.a2a_agent import A2AAgent from agent_env.bundle import BundleKind, parse_bundle from agent_env.bundle.installed import CORE, installed_bundles from agent_env.bundle.plan import check_bundle, plan_bundle from agent_env.bundle.resolve import resolve_bundle from agent_env.config import get_config -from agent_env.plugins._bundles import installed_aliases +from agent_env.plugins._bundles import ENV_ALIASES, installed_aliases from tst.unit.bundle._support import layout @@ -25,7 +26,7 @@ def _shipped() -> dict: def test_agentenv_framework_ships_its_bundles(): - assert SHIPPED == ["agents", "echo-mcp", "hello"] + assert SHIPPED == ["agents", "echo-mcp", "hello", "mail-mcp"] @pytest.mark.parametrize("name", SHIPPED) @@ -41,13 +42,22 @@ def test_the_shipped_aliases_are_agent_envs_alone_and_claude_code_names_the_defa table = installed_aliases(tuple(_shipped().values())) assert table.refused == {} and table.refused_claims == {} - assert {alias: named.agent_id for alias, named in table.aliases.items()} == { + assert {alias: named.target_id for alias, named in table.aliases.items()} == { "claude-code": "@local/agentenv-framework/agents/claude-code", "pi": "@local/agentenv-framework/agents/pi", "openhands": "@local/agentenv-framework/agents/openhands", "codex": "@local/agentenv-framework/agents/codex", "gemini": "@local/agentenv-framework/agents/gemini"} - assert table.target("claude-code").agent_id == get_config().get_default_a2a_agent_id() + assert table.target("claude-code").target_id == get_config().get_default_a2a_agent_id() + +def test_the_shipped_env_alias_names_the_mail_mcp_examples_env_which_the_validator_deploys(): + table = installed_aliases(tuple(_shipped().values()), ENV_ALIASES) + + assert table.refused == {} and table.refused_claims == {} + assert {alias: named.target_id for alias, named in table.aliases.items()} == { + "mail-mcp": "@local/agentenv-framework/mail-mcp/mail"} + assert A2AAgent.VALIDATION_ENV_ID == "mail-mcp" + def test_a_task_naming_no_agent_plans_with_the_shipped_default(tmp_path, monkeypatch, local_stores, cli_routing): monkeypatch.setenv("HOME", str(tmp_path)) layout(tmp_path / "bare", {"tasks/t.json": json.dumps([{"id": "agent", "type": "deploy_agent", "env_ids": []}])}) diff --git a/tst/unit/cli/a2a_agent_validate_test.py b/tst/unit/cli/a2a_agent_validate_test.py new file mode 100644 index 00000000..51bb841f --- /dev/null +++ b/tst/unit/cli/a2a_agent_validate_test.py @@ -0,0 +1,72 @@ +"""`agent-env a2a-agent validate` writes the env it deploys, the mail-mcp example's by its alias, when the store has +none of that id.""" + +import importlib + +import pytest +from click.testing import CliRunner + +from agent_env.a2a_agent import A2AAgent +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.artifact.store import get_artifact_store +from agent_env.bundle import BundleError +from agent_env.cli import cli +from agent_env.env.envs.mcp_server import MCPServerEnv + +# By its path: the package exports the command under the module's name. +validate_module = importlib.import_module("agent_env.cli.a2a_agent.validate") + + +@pytest.fixture +def written(monkeypatch, local_stores, cli_routing): + """The bundles validate writes, as (root, id_root, sandbox).""" + calls = [] + monkeypatch.setattr(validate_module, "write_bundle", lambda root, **kwargs: calls.append( + (root.name, kwargs["id_root"], kwargs["sandbox"]))) + return calls + + +def test_the_validator_deploys_the_mail_mcp_examples_env_by_its_alias(): + assert A2AAgent.VALIDATION_ENV_ID == "mail-mcp" + + +def test_validate_writes_the_example_its_alias_names_when_the_store_has_no_env_of_that_id(written): + validate_module._write_validation_env("local") + + assert written == [("mail-mcp", "@local/agentenv-framework/mail-mcp", "local")] + + +def test_the_stores_env_of_that_id_is_the_one_validate_deploys(written): + image = get_artifact_store().put_document(DockerImageArtifact( + id="mail-mcp/env_image", description="d", image_name="ghcr.io/acme/mail:1", tar_gz_s3_url="s3://b/x")) + MCPServerEnv.put(id="mail-mcp", docker_image_artifact=image, environment_name="mail") + + validate_module._write_validation_env(None) + + assert written == [] + + +def test_stores_that_arent_local_need_an_env_of_that_id_and_are_told_how_to_put_one(written, monkeypatch): + monkeypatch.setattr(validate_module, "stores_are_local", lambda: False) + + with pytest.raises(BundleError, match="the store has no env 'mail-mcp', the one validate deploys, and validate " + "writes agentenv-framework's mail-mcp bundle's only into local stores; put " + r"it with `agent-env env mcp-server put --id mail-mcp --dockerfile .*" + r"mail-mcp/envs/mail/Dockerfile`"): + validate_module._write_validation_env(None) + assert written == [] + + +def test_the_cli_writes_the_env_before_it_validates(written, monkeypatch): + image = get_artifact_store().put_document(DockerImageArtifact( + id="agent-image", description="d", image_name="ghcr.io/acme/agent:1", tar_gz_s3_url="s3://b/x")) + A2AAgent.put(id="mine", docker_image_artifact=image) + order = [] + monkeypatch.setattr(validate_module, "_write_validation_env", lambda sandbox: order.append("write")) + monkeypatch.setattr(validate_module, "run_validation", lambda agent, litellm_api_key=None: order.append( + "validate") or {"a2a_agent_mcp": {"passed": True}}) + + result = CliRunner().invoke(cli, ["a2a-agent", "validate", "--id", "mine"]) + + assert result.exit_code == 0, result.output + assert order == ["write", "validate"] diff --git a/tst/unit/env/alias_get_test.py b/tst/unit/env/alias_get_test.py new file mode 100644 index 00000000..56029e36 --- /dev/null +++ b/tst/unit/env/alias_get_test.py @@ -0,0 +1,94 @@ +"""Env.get: the store's env of an id, else the env an installed bundle's alias of that id names, under the alias's +id.""" + +import pytest + +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.artifact.store import get_artifact_store +from agent_env.env import env as env_module +from agent_env.env.env import Env +from agent_env.env.envs.mcp_server import MCPServerEnv +from agent_env.plugins._bundles import ALIASES_FILE, InstalledBundle, installed_aliases +from agent_env.store.base import NotFoundError +from agent_env.store.routing import namespace_routing + +SHIPPED = "@local/agentenv-framework/mail-mcp/mail" + + +@pytest.fixture +def bundles(tmp_path, monkeypatch, local_stores): + """agent-env's mail-mcp bundle gives its env the alias mail-mcp, and two plugin bundles both give helper.""" + installed = [] + for package, name, toml in [("agentenv-framework", "mail-mcp", '[envs]\nmail = ["mail-mcp"]\n'), + ("acme-kit", "kit", '[envs]\na = ["helper"]\n'), + ("zeta-tools", "tools", '[envs]\nb = ["helper"]\n')]: + root = tmp_path / package / name + root.mkdir(parents=True) + (root / ALIASES_FILE).write_text(toml) + installed.append(InstalledBundle(name, package, "1.0", package, root, None, None)) + monkeypatch.setattr(env_module, "installed_aliases", lambda table: installed_aliases(installed, table)) + + +@pytest.fixture +def aliases(bundles, cli_routing): + """``bundles``, with namespace routing on, as in the CLI.""" + + +def _put(env_id): + image = get_artifact_store().put_document(DockerImageArtifact( + id=f"{env_id}/env_image", description="d", image_name="ghcr.io/acme/mail:1", tar_gz_s3_url="s3://b/x")) + return MCPServerEnv.put(id=env_id, docker_image_artifact=image, environment_name="mail") + + +def test_an_alias_names_the_env_its_bundle_gives_it_under_the_aliass_id(aliases): + _put(SHIPPED) + _put(SHIPPED) + + env = Env.get("mail-mcp") + + assert (env.id, env.version, env.environment_name) == ("mail-mcp", 2, "mail") + assert Env.get("mail-mcp", 1).version == 1 # a deployment's record reloads the version it deployed + assert Env.get(SHIPPED).id == SHIPPED + + +def test_the_stores_env_of_an_aliass_id_is_the_one_it_names(aliases): + _put(SHIPPED) + _put("mail-mcp") + + assert Env.get("mail-mcp").docker_image_artifact.id == "mail-mcp/env_image" + + +def test_a_version_the_stores_env_of_an_alias_id_lacks_is_not_found(aliases): + _put(SHIPPED) + _put(SHIPPED) + _put("mail-mcp") + + with pytest.raises(NotFoundError): + Env.get("mail-mcp", 2) + + +def test_an_alias_whose_env_no_run_has_written_says_so(aliases): + with pytest.raises(NotFoundError, match=f"'mail-mcp' is an alias of {SHIPPED}, which comes with " + "agentenv-framework's mail-mcp bundle and isn't in the store yet"): + Env.get("mail-mcp") + + +def test_an_alias_two_bundles_give_is_refused_naming_both(aliases): + with pytest.raises(NotFoundError, match="'helper' is an alias acme-kit's kit bundle and zeta-tools's tools bundle " + "each give an env"): + Env.get("helper") + + +def test_an_id_close_to_an_alias_points_at_it(aliases): + with pytest.raises(NotFoundError, match="there is no env 'mail_mcp' in the store; did you mean the alias " + "'mail-mcp'"): + Env.get("mail_mcp") + + +def test_without_namespace_routing_an_alias_names_no_env_and_says_to_put_one(bundles): + with namespace_routing(): + _put(SHIPPED) + + with pytest.raises(NotFoundError, match="an alias names its env only in the CLI; put an env of the id " + "'mail-mcp' in the store"): + Env.get("mail-mcp") diff --git a/tst/unit/test_clean_install_wheel.py b/tst/unit/test_clean_install_wheel.py index 41313f38..5427892a 100644 --- a/tst/unit/test_clean_install_wheel.py +++ b/tst/unit/test_clean_install_wheel.py @@ -63,4 +63,4 @@ def test_the_gate_checks_the_folder_of_every_shipped_bundle(): roots = {name: root.relative_to(SCRIPT.parents[2]).as_posix() for name, root in clean_install.bundle_roots().items()} assert roots == {"agents": "src/agent_env/agents", "echo-mcp": "src/agent_env/examples/echo-mcp", - "hello": "src/agent_env/examples/hello"} + "hello": "src/agent_env/examples/hello", "mail-mcp": "src/agent_env/examples/mail-mcp"} diff --git a/tst/unit/test_release_protocol_pins.py b/tst/unit/test_release_protocol_pins.py index b8fd1b96..09d765dc 100644 --- a/tst/unit/test_release_protocol_pins.py +++ b/tst/unit/test_release_protocol_pins.py @@ -88,6 +88,7 @@ def test_each_shipped_requirements_file_the_release_pins_holds_one_protocol_line for path in shipped_bundles.requirements_files()} assert "src/agent_env/examples/echo-mcp/envs/echo/requirements.txt" in pinned + assert "src/agent_env/examples/mail-mcp/envs/mail/requirements.txt" in pinned assert all(len(lines) == 1 for lines in pinned.values()), pinned From 0441b9fb060a9fd5db22c1ea89039341e29c14a1 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 23 Oct 2026 14:31:27 -0700 Subject: [PATCH 02/10] fix(env): an env got by an alias writes its metadata to the env the alias names Env.get gave an alias's env under the alias's id, so update_metadata and merge_metadata wrote to an id the store doesn't hold. The env keeps the id it's stored under (stored_id) for them, and the alias for the runs that name it. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/agent_env/env/env.py | 13 +++++++++---- tst/unit/env/alias_get_test.py | 11 +++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/agent_env/env/env.py b/src/agent_env/env/env.py index a429af91..f97d0661 100644 --- a/src/agent_env/env/env.py +++ b/src/agent_env/env/env.py @@ -269,9 +269,14 @@ def get(cls, id: str, version: Optional[int] = None) -> "Env": raise NotFoundError(f"{id!r} is an alias of {named.target_id}, which comes with {named.declared_by} and " "isn't in the store yet: an `agent-env run` of a bundle that uses it writes it") from None env = copy.copy(env) - env.id = id + env._stored_id, env.id = env.id, id return env + @property + def stored_id(self) -> str: + """The id the store holds this env under: its own, or for an env ``get`` gave by an alias, the aliased env's.""" + return getattr(self, "_stored_id", None) or self.id + @classmethod async def from_instance_id(cls, instance_id: str) -> "Env": from .store import get_env_instance_store @@ -304,7 +309,7 @@ def update_metadata(self, new_metadata: dict[str, Any]) -> None: from .store import get_env_store if self.version is None: raise ValueError("Cannot update metadata on an unsaved env (version is None)") - self.metadata = get_env_store().update_metadata(self.id, self.version, self.metadata, new_metadata) + self.metadata = get_env_store().update_metadata(self.stored_id, self.version, self.metadata, new_metadata) def merge_metadata(self, updates: dict[str, Any], retries: int = 3) -> None: """Merge ``updates`` into this env's metadata, re-reading and retrying on a @@ -319,13 +324,13 @@ def merge_metadata(self, updates: dict[str, Any], retries: int = 3) -> None: for attempt in range(retries + 1): try: self.metadata = get_env_store().update_metadata( - self.id, self.version, metadata, {**metadata, **updates} + self.stored_id, self.version, metadata, {**metadata, **updates} ) return except ConcurrentModificationError: if attempt == retries: raise - metadata = Env.get(self.id, self.version).metadata + metadata = Env.get(self.stored_id, self.version).metadata def to_dict(self) -> dict[str, Any]: return {"id": self.id, "type": self.type, "version": self.version, "metadata": self.metadata} diff --git a/tst/unit/env/alias_get_test.py b/tst/unit/env/alias_get_test.py index 56029e36..2483c1a7 100644 --- a/tst/unit/env/alias_get_test.py +++ b/tst/unit/env/alias_get_test.py @@ -51,6 +51,17 @@ def test_an_alias_names_the_env_its_bundle_gives_it_under_the_aliass_id(aliases) assert Env.get(SHIPPED).id == SHIPPED + +def test_an_env_got_by_an_alias_writes_its_metadata_to_the_env_the_alias_names(aliases): + _put(SHIPPED) + env = Env.get("mail-mcp") + + env.update_metadata({**env.metadata, "checked": "yes"}) + env.merge_metadata({"merged": "too"}) + + assert (env.id, env.stored_id) == ("mail-mcp", SHIPPED) + assert {key: Env.get(SHIPPED).metadata[key] for key in ("checked", "merged")} == {"checked": "yes", "merged": "too"} + def test_the_stores_env_of_an_aliass_id_is_the_one_it_names(aliases): _put(SHIPPED) _put("mail-mcp") From aa666bd3c4b8e25a3f8afa76d95ada93bad31ed1 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 23 Oct 2026 15:21:51 -0700 Subject: [PATCH 03/10] fix(validate): check the mail-mcp example on the env backend the validation deploys it on --sandbox picks the agents' backend; the validation deploys the env on the configured env backend, so writing the example checked the wrong one. The installer journey also lists agent-env's four bundles by count now. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/agent_env/cli/a2a_agent/validate.py | 9 +++++---- tst/installer/test_container_journey.py | 2 +- tst/unit/cli/a2a_agent_validate_test.py | 19 ++++++++++--------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/agent_env/cli/a2a_agent/validate.py b/src/agent_env/cli/a2a_agent/validate.py index f9a0bbae..2968b0f4 100644 --- a/src/agent_env/cli/a2a_agent/validate.py +++ b/src/agent_env/cli/a2a_agent/validate.py @@ -56,7 +56,7 @@ def validate(agent_id: str, agent_version: int | None, litellm_api_key: str | No sys.exit(1) try: - _write_validation_env(sandbox) + _write_validation_env() except (BundleError, InfraError, DockerBuildError) as e: click.echo(click.style(f"Error: {e}", fg="red"), err=True) sys.exit(1) @@ -66,9 +66,10 @@ def validate(agent_id: str, agent_version: int | None, litellm_api_key: str | No sys.exit(1) -def _write_validation_env(sandbox: str | None) -> None: +def _write_validation_env() -> None: """Write the env the validator deploys, ``A2AAgent.VALIDATION_ENV_ID``, when the store has none of that id: the env - an installed bundle gives that alias, with the infra it deploys on, into local stores. Stores that aren't local + an installed bundle gives that alias, with the infra it deploys on, into local stores, checked on the env backend + the validation deploys it on, the configured one (``--sandbox`` picks the agents'). Stores that aren't local need an env of that id of their own, and BundleError says how to put one.""" env_id = A2AAgent.VALIDATION_ENV_ID try: @@ -85,4 +86,4 @@ def _write_validation_env(sandbox: str | None) -> None: f"{named.declared_by}'s only into local stores; put it with `agent-env env mcp-server put " f"--id {env_id} --dockerfile {dockerfile}`"]) click.echo(f"{env_id} → {named.target_id} (the alias {named.declared_by} gives it): writing it") - write_bundle(named.bundle.root, id_root=named.bundle.id_root, sandbox=sandbox, on_progress=click.echo) + write_bundle(named.bundle.root, id_root=named.bundle.id_root, on_progress=click.echo) diff --git a/tst/installer/test_container_journey.py b/tst/installer/test_container_journey.py index 2a61d796..02aa69ad 100644 --- a/tst/installer/test_container_journey.py +++ b/tst/installer/test_container_journey.py @@ -133,7 +133,7 @@ def test_a_new_user_installs_agent_env_adds_plugins_runs_a_task_and_removes_them container.ok("python", "-m", "pip", "install", "-q", "--no-index", "--find-links", "/wheels", "uv") container.ok("uv", "tool", "install", "agentenv-framework") rows = container.ok("agent-env", "plugin", "list").stdout.split("STATUS\n", 1)[1] - assert re.fullmatch(r"agentenv-framework +\S+ +bundle agents, bundle echo-mcp, bundle hello +ok\n", rows), rows + assert re.fullmatch(r"agentenv-framework +\S+ +4 bundles +ok\n", rows), rows added = container.ok("agent-env", "plugin", "add", browser, grader, "--yes") assert "toy_navigate active" in added.stdout and "toy_grade active" in added.stdout diff --git a/tst/unit/cli/a2a_agent_validate_test.py b/tst/unit/cli/a2a_agent_validate_test.py index 51bb841f..72c0bc3c 100644 --- a/tst/unit/cli/a2a_agent_validate_test.py +++ b/tst/unit/cli/a2a_agent_validate_test.py @@ -19,10 +19,10 @@ @pytest.fixture def written(monkeypatch, local_stores, cli_routing): - """The bundles validate writes, as (root, id_root, sandbox).""" + """The bundles validate writes, as (root, id_root, the sandbox each is checked on).""" calls = [] monkeypatch.setattr(validate_module, "write_bundle", lambda root, **kwargs: calls.append( - (root.name, kwargs["id_root"], kwargs["sandbox"]))) + (root.name, kwargs["id_root"], kwargs.get("sandbox")))) return calls @@ -30,10 +30,11 @@ def test_the_validator_deploys_the_mail_mcp_examples_env_by_its_alias(): assert A2AAgent.VALIDATION_ENV_ID == "mail-mcp" -def test_validate_writes_the_example_its_alias_names_when_the_store_has_no_env_of_that_id(written): - validate_module._write_validation_env("local") +def test_validate_writes_the_example_its_alias_names_on_the_env_backend_when_the_store_has_no_env_of_that_id(written): + """Checked where the validation deploys the env, the configured env backend, not on --sandbox, the agents'.""" + validate_module._write_validation_env() - assert written == [("mail-mcp", "@local/agentenv-framework/mail-mcp", "local")] + assert written == [("mail-mcp", "@local/agentenv-framework/mail-mcp", None)] def test_the_stores_env_of_that_id_is_the_one_validate_deploys(written): @@ -41,7 +42,7 @@ def test_the_stores_env_of_that_id_is_the_one_validate_deploys(written): id="mail-mcp/env_image", description="d", image_name="ghcr.io/acme/mail:1", tar_gz_s3_url="s3://b/x")) MCPServerEnv.put(id="mail-mcp", docker_image_artifact=image, environment_name="mail") - validate_module._write_validation_env(None) + validate_module._write_validation_env() assert written == [] @@ -53,7 +54,7 @@ def test_stores_that_arent_local_need_an_env_of_that_id_and_are_told_how_to_put_ "writes agentenv-framework's mail-mcp bundle's only into local stores; put " r"it with `agent-env env mcp-server put --id mail-mcp --dockerfile .*" r"mail-mcp/envs/mail/Dockerfile`"): - validate_module._write_validation_env(None) + validate_module._write_validation_env() assert written == [] @@ -62,11 +63,11 @@ def test_the_cli_writes_the_env_before_it_validates(written, monkeypatch): id="agent-image", description="d", image_name="ghcr.io/acme/agent:1", tar_gz_s3_url="s3://b/x")) A2AAgent.put(id="mine", docker_image_artifact=image) order = [] - monkeypatch.setattr(validate_module, "_write_validation_env", lambda sandbox: order.append("write")) + monkeypatch.setattr(validate_module, "_write_validation_env", lambda: order.append("write")) monkeypatch.setattr(validate_module, "run_validation", lambda agent, litellm_api_key=None: order.append( "validate") or {"a2a_agent_mcp": {"passed": True}}) - result = CliRunner().invoke(cli, ["a2a-agent", "validate", "--id", "mine"]) + result = CliRunner().invoke(cli, ["a2a-agent", "validate", "--id", "mine", "--sandbox", "local"]) assert result.exit_code == 0, result.output assert order == ["write", "validate"] From 7a35c89b83de3e05b8e44011ef31f53c43a97973 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 23 Oct 2026 16:13:37 -0700 Subject: [PATCH 04/10] fix(env): reload an aliased env's deployment from the env it was, and refuse env aliases in env definitions A deployment of an env got by an alias recorded only the alias, so a reload after the store gained an env of that id read the wrong one. Its record now keeps the id the env is stored under, and the reloads go through Env.of_deployment, which reads that env under the alias. An env definition naming an env alias, as a MultiEnv's servers might, is refused with what to name instead, since an alias names an env only in a task's steps. Co-Authored-By: Claude Opus 5.5 (1M context) --- AGENTS.md | 6 +++-- src/agent_env/bundle/plan.py | 4 +++ src/agent_env/env/env.py | 26 ++++++++++++++++--- src/agent_env/env/envs/_deployment.py | 4 ++- src/agent_env/env/envs/mcp_server.py | 2 +- src/agent_env/env/envs/multi_env.py | 2 +- src/agent_env/env/envs/website.py | 2 +- .../snapshot_utils/snapshot_series.py | 2 +- .../task_steps/apply_server_config.py | 2 +- .../task_step/task_steps/load_artifact.py | 4 +-- .../verify_universe_roundtrip.py | 2 +- .../task_step/task_steps/reset_env.py | 2 +- .../task_step/task_steps/sync_env_clock.py | 2 +- .../cli/run_bundle_built_envs_test.py | 1 + tst/unit/bundle/installed_entities_test.py | 11 ++++++++ tst/unit/env/alias_get_test.py | 14 +++++++++- 16 files changed, 68 insertions(+), 18 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6097587e..296afb71 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -160,8 +160,10 @@ Out-of-tree code plugs in either through `config.toml`, as a `module:Class` poin `step_params` overrides, so the tasks stay as written. - `aliases.toml` gives envs aliases the same way (`[envs] = ["", ...]`; the `mail-mcp` example names its env `mail-mcp`), resolved by the planner (`Plan.env_aliases`) and by `Env.get`. The env `Env.get` gives for an - alias answers to the alias's id, so a deployment records, and steps look it up by, the id the task names; an alias - names its env only where namespace routing is on (the CLI), so a service needs an env of the alias's id in its store. + alias answers to the alias's id, so a deployment records, and steps look it up by, the id the task names; its record + also keeps the id the env is stored under (`STORED_ENV_ID_KEY`), which `Env.of_deployment` reloads it from. An alias + names its env only where namespace routing is on (the CLI), so a service needs an env of the alias's id in its store, + and only in a task's steps: an env definition (a MultiEnv's servers) naming one is refused. `agent-env a2a-agent validate` deploys `mail-mcp` (`A2AAgent.VALIDATION_ENV_ID`), writing the example's env with `bundle.write_bundle` (a bundle's writes and infra, no run) when the store has none of that id. - Environment providers have no table: an installed package registers one as an `agent_env.env_providers` diff --git a/src/agent_env/bundle/plan.py b/src/agent_env/bundle/plan.py index 91c6ae16..853f7c2d 100644 --- a/src/agent_env/bundle/plan.py +++ b/src/agent_env/bundle/plan.py @@ -700,8 +700,12 @@ def _store_ref_problem(self, ref: Reference, read: dict) -> str | None: if found is None: at = f" version {ref.version}" if ref.version is not None else "" other = self.named.get(fold(ref.id)) + aliased = self.env_alias_table.target(ref.id) if ref.kind is EntityKind.ENV and self.env_alias_table else None if other is not None: hint = f"; this bundle's {other.kind.store} {other.name!r} has that name" + elif aliased is not None: # a task's reference to it is pulled in; another entry's isn't, as aliases go + return (f"{ref.id!r} is an alias of {aliased.target_id}, which comes with {aliased.declared_by}, and an " + "env alias names an env only in a task's steps; name the env by its id") else: table = {EntityKind.AGENT: self.alias_table, EntityKind.ENV: self.env_alias_table}.get(ref.kind) hint = table.hint(ref.id) if table is not None else "" diff --git a/src/agent_env/env/env.py b/src/agent_env/env/env.py index f97d0661..fe38adfb 100644 --- a/src/agent_env/env/env.py +++ b/src/agent_env/env/env.py @@ -64,6 +64,17 @@ def __init__(self, what: str, env_id: str) -> None: super().__init__(f"{what} needs a sandbox; env '{env_id}' runs outside agent-env's sandboxes") +# The deployment record of an env got by an alias names the env it was here, as the alias names it now may differ. +STORED_ENV_ID_KEY = "stored_env_id" + + +def _aliased(env: "Env", alias: str) -> "Env": + """A copy of ``env`` answering to ``alias``, which names it, while it keeps the id the store holds it under.""" + env = copy.copy(env) + env._stored_id, env.id = env.id, alias + return env + + def _holds(env_id: str) -> bool: """Whether the store has an env of ``env_id``, at any version.""" try: @@ -268,9 +279,16 @@ def get(cls, id: str, version: Optional[int] = None) -> "Env": except NotFoundError: raise NotFoundError(f"{id!r} is an alias of {named.target_id}, which comes with {named.declared_by} and " "isn't in the store yet: an `agent-env run` of a bundle that uses it writes it") from None - env = copy.copy(env) - env._stored_id, env.id = env.id, id - return env + return _aliased(env, id) + + @classmethod + def of_deployment(cls, deployed: "DeployedEnv") -> "Env": + """The env ``deployed`` deployed, at its version: the store's env of its id, or for an env got by an alias, the + env the alias named then (``STORED_ENV_ID_KEY``), still under the alias's id, whatever the store holds now.""" + stored = (getattr(deployed, "metadata", None) or {}).get(STORED_ENV_ID_KEY) + if stored is None: + return cls.get(deployed.env_id, deployed.env_version) + return _aliased(get_env_store().get(stored, deployed.env_version), deployed.env_id) @property def stored_id(self) -> str: @@ -281,7 +299,7 @@ def stored_id(self) -> str: async def from_instance_id(cls, instance_id: str) -> "Env": from .store import get_env_instance_store deployed = get_env_instance_store().get(instance_id) - env = cls.get(deployed.env_id, deployed.env_version) + env = cls.of_deployment(deployed) return await type(env).from_deployed_env(deployed) @classmethod diff --git a/src/agent_env/env/envs/_deployment.py b/src/agent_env/env/envs/_deployment.py index 2d4db4d7..70b89c3a 100644 --- a/src/agent_env/env/envs/_deployment.py +++ b/src/agent_env/env/envs/_deployment.py @@ -10,7 +10,7 @@ from agentenv_protocol import METHOD_ADD, METHOD_RESET, RPC_PATH, FilePart, client as protocol_v1 from agent_env.env import legacy_protocol -from agent_env.env.env import DeployedSandboxEnv, gateway_url_of +from agent_env.env.env import STORED_ENV_ID_KEY, DeployedSandboxEnv, gateway_url_of from agent_env.env.store import register_env_instance from agent_env.store.base import ObjectNotFoundError @@ -66,6 +66,8 @@ async def deploy_through_provider(env: Env, *, environment_name: str | None, ttl else: check_plugin_record(env, deployed) env._gateway_url = gateway_url_of(deployed) + if env.stored_id != env.id: # got by an alias: the record keeps the env it was, for a reload + deployed.metadata = {**(deployed.metadata or {}), STORED_ENV_ID_KEY: env.stored_id} deployed = register_env_instance(deployed, ttl_seconds) env._instance_id = deployed.instance_id env._deployed, env._replaced = deployed, replaced diff --git a/src/agent_env/env/envs/mcp_server.py b/src/agent_env/env/envs/mcp_server.py index 46128d48..ea345ba5 100644 --- a/src/agent_env/env/envs/mcp_server.py +++ b/src/agent_env/env/envs/mcp_server.py @@ -118,7 +118,7 @@ def deploy_refusal(self, **options) -> str | None: @classmethod async def from_deployed_env(cls, deployed: DeployedEnv) -> MCPServerEnv: - env = Env.get(deployed.env_id, deployed.env_version) + env = Env.of_deployment(deployed) if not isinstance(env, MCPServerEnv): raise TypeError(f"Expected MCPServerEnv, got {type(env).__name__}") if env._env_provider is None: diff --git a/src/agent_env/env/envs/multi_env.py b/src/agent_env/env/envs/multi_env.py index ec163dc5..f215da13 100644 --- a/src/agent_env/env/envs/multi_env.py +++ b/src/agent_env/env/envs/multi_env.py @@ -151,7 +151,7 @@ def deploy_refusal(self, **options) -> str | None: @classmethod async def from_deployed_env(cls, deployed: DeployedEnv) -> MultiEnv: - env = Env.get(deployed.env_id, deployed.env_version) + env = Env.of_deployment(deployed) if not isinstance(env, MultiEnv): raise TypeError(f"Expected MultiEnv, got {type(env).__name__}") provider = builtin_provider_for(env.env_provider_type) or plugin_provider_like_a_builtin(env.env_provider_type) diff --git a/src/agent_env/env/envs/website.py b/src/agent_env/env/envs/website.py index a35dd30e..ceb24080 100644 --- a/src/agent_env/env/envs/website.py +++ b/src/agent_env/env/envs/website.py @@ -138,7 +138,7 @@ def deploy_refusal(self, **options) -> str | None: @classmethod async def from_deployed_env(cls, deployed: DeployedEnv) -> WebsiteEnv: - env = Env.get(deployed.env_id, deployed.env_version) + env = Env.of_deployment(deployed) if not isinstance(env, WebsiteEnv): raise TypeError(f"Expected WebsiteEnv, got {type(env).__name__}") if env._env_provider is None: diff --git a/src/agent_env/task_step/snapshot_utils/snapshot_series.py b/src/agent_env/task_step/snapshot_utils/snapshot_series.py index f0b8817e..780340ee 100644 --- a/src/agent_env/task_step/snapshot_utils/snapshot_series.py +++ b/src/agent_env/task_step/snapshot_utils/snapshot_series.py @@ -502,7 +502,7 @@ async def _capture_env_state( try: # Pin the env version the sandbox actually runs, so service # enumeration can't drift if the env was re-registered since deploy. - env = await asyncio.to_thread(Env.get, deployed.env_id, deployed.env_version) + env = await asyncio.to_thread(Env.of_deployment, deployed) result = await SnapshotEnvTaskStep.snapshot_env_state( env=env, gateway_url=deployed.gateway_url, diff --git a/src/agent_env/task_step/task_steps/apply_server_config.py b/src/agent_env/task_step/task_steps/apply_server_config.py index d1495fb9..598be40e 100644 --- a/src/agent_env/task_step/task_steps/apply_server_config.py +++ b/src/agent_env/task_step/task_steps/apply_server_config.py @@ -143,7 +143,7 @@ def _expand_directives(self, deployed) -> list[ConfigDirective]: from agent_env.env.env import Env - env = Env.get(deployed.env_id, deployed.env_version) + env = Env.of_deployment(deployed) environment_names = [ e.environment_name for e in (getattr(env, "mcp_server_envs", None) or []) ] diff --git a/src/agent_env/task_step/task_steps/load_artifact.py b/src/agent_env/task_step/task_steps/load_artifact.py index 1555c9ae..b42458ab 100644 --- a/src/agent_env/task_step/task_steps/load_artifact.py +++ b/src/agent_env/task_step/task_steps/load_artifact.py @@ -499,7 +499,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: # Only a FileArtifactUniverse or a FileArtifact reaches here — `stage_as_files` # excludes a service universe whenever an env is targeted. if env is None: - env_obj = Env.get(deployed.env_id, deployed.env_version) + env_obj = Env.of_deployment(deployed) env = await type(env_obj).from_deployed_env(deployed) result = await env.load_file_artifact_universe(artifact, destination_path=destination_path) context.metadata.setdefault("loaded_file_artifact_universes", []).append({ @@ -583,7 +583,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: if deployed is None: raise RuntimeError(f"`env_id` is required when loading a {artifact.type}") if env is None: - env_obj = Env.get(deployed.env_id, deployed.env_version) + env_obj = Env.of_deployment(deployed) env = await type(env_obj).from_deployed_env(deployed) if isinstance(artifact, EnvironmentArtifact): await env.load_environment_artifact(artifact) diff --git a/src/agent_env/task_step/task_steps/multienv_validator/verify_universe_roundtrip.py b/src/agent_env/task_step/task_steps/multienv_validator/verify_universe_roundtrip.py index 3e5b6b3d..f5f0257a 100644 --- a/src/agent_env/task_step/task_steps/multienv_validator/verify_universe_roundtrip.py +++ b/src/agent_env/task_step/task_steps/multienv_validator/verify_universe_roundtrip.py @@ -102,7 +102,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: deployed = next((d for d in context.deployed_envs if d.env_id == self.env_id), None) if deployed is None: raise RuntimeError(f"Env '{self.env_id}' not found in context.deployed_envs") - env = Env.get(deployed.env_id, deployed.env_version) + env = Env.of_deployment(deployed) env = await type(env).from_deployed_env(deployed) universe = EnvironmentUniverseArtifact.get(self.universe_artifact_id, self.universe_artifact_version) environment_artifacts = universe.get_environment_artifacts() diff --git a/src/agent_env/task_step/task_steps/reset_env.py b/src/agent_env/task_step/task_steps/reset_env.py index 88059fdd..fee656d7 100644 --- a/src/agent_env/task_step/task_steps/reset_env.py +++ b/src/agent_env/task_step/task_steps/reset_env.py @@ -58,7 +58,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: logger.warning("reset_env: no deployed env '%s' in context; skipping", self.env_id) return context try: - env = Env.get(deployed.env_id, deployed.env_version) + env = Env.of_deployment(deployed) await env.reset(deployed) logger.info("reset_env: reset env '%s'", self.env_id) except NotImplementedError: diff --git a/src/agent_env/task_step/task_steps/sync_env_clock.py b/src/agent_env/task_step/task_steps/sync_env_clock.py index 1dc48a55..41ba3707 100644 --- a/src/agent_env/task_step/task_steps/sync_env_clock.py +++ b/src/agent_env/task_step/task_steps/sync_env_clock.py @@ -72,7 +72,7 @@ def from_dict(cls, data: dict) -> SyncEnvClockTaskStep: def _service_names(self, deployed) -> list[str]: """Backing MCP server names for the env (mirrors apply_server_config's enumeration).""" from agent_env.env.env import Env - env = Env.get(deployed.env_id, deployed.env_version) + env = Env.of_deployment(deployed) names = [e.environment_name for e in (getattr(env, "mcp_server_envs", None) or [])] if not names and getattr(env, "environment_name", None): names = [env.environment_name] diff --git a/tst/integration/cli/run_bundle_built_envs_test.py b/tst/integration/cli/run_bundle_built_envs_test.py index 9163312b..8aa43698 100644 --- a/tst/integration/cli/run_bundle_built_envs_test.py +++ b/tst/integration/cli/run_bundle_built_envs_test.py @@ -581,6 +581,7 @@ def test_the_shipped_mail_mcp_example_runs_and_a_task_outside_a_bundle_deploys_i [written] = (state / "out").iterdir() [deployed] = json.loads(written.read_text())["deployed_envs"] assert deployed["env_id"] == "mail-mcp" and deployed["gateway_url"], deployed + assert deployed["metadata"]["stored_env_id"] == "@local/agentenv-framework/mail-mcp/mail", deployed ECHO = REPO / "src" / "agent_env" / "examples" / "echo-mcp" diff --git a/tst/unit/bundle/installed_entities_test.py b/tst/unit/bundle/installed_entities_test.py index 8eeb3c26..f26cf2e8 100644 --- a/tst/unit/bundle/installed_entities_test.py +++ b/tst/unit/bundle/installed_entities_test.py @@ -396,6 +396,17 @@ def test_the_stores_env_of_an_alias_id_shadows_the_alias(bundle_dir, tmp_path): assert plan.env_aliases["crm-x"].store_version == 1 and [ref.id for ref in plan.store_refs] == ["crm-x"] + +def test_an_env_definition_naming_an_env_alias_is_refused_saying_to_name_the_env_by_its_id(bundle_dir, tmp_path): + """An env alias names an env in a task's steps; a MultiEnv's servers name theirs by id.""" + kit = _kit(tmp_path, ENV_ALIASED) + + problems = _problems(_task(bundle_dir, [{"id": "env", "type": "deploy_env", "env_id": "suite"}], + {"envs/suite/env.toml": 'type = "multi"\nmcp_server_envs = ["crm-x"]\n'}), kit) + + assert any(f"'crm-x' is an alias of {CRM}, which comes with demo-dist's kit bundle, and an env alias names an env only " + "in a task's steps; name the env by its id" in problem for problem in problems), problems + def test_the_stores_agent_of_an_alias_id_shadows_the_alias(bundle_dir, tmp_path): """Someone's own agent of that id is the one they mean.""" kit = _kit(tmp_path, ALIASED) diff --git a/tst/unit/env/alias_get_test.py b/tst/unit/env/alias_get_test.py index 2483c1a7..7c20294e 100644 --- a/tst/unit/env/alias_get_test.py +++ b/tst/unit/env/alias_get_test.py @@ -6,7 +6,7 @@ from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.store import get_artifact_store from agent_env.env import env as env_module -from agent_env.env.env import Env +from agent_env.env.env import STORED_ENV_ID_KEY, DeployedEnv, Env from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.plugins._bundles import ALIASES_FILE, InstalledBundle, installed_aliases from agent_env.store.base import NotFoundError @@ -62,6 +62,18 @@ def test_an_env_got_by_an_alias_writes_its_metadata_to_the_env_the_alias_names(a assert (env.id, env.stored_id) == ("mail-mcp", SHIPPED) assert {key: Env.get(SHIPPED).metadata[key] for key in ("checked", "merged")} == {"checked": "yes", "merged": "too"} + +def test_a_deployment_of_an_env_got_by_an_alias_reloads_that_env_whatever_the_store_holds_since(aliases): + """A record of the alias's env names the env it was, so one the store gains under the alias's id later isn't it.""" + _put(SHIPPED) + deployed = DeployedEnv(env_id="mail-mcp", env_version=1, metadata={STORED_ENV_ID_KEY: SHIPPED}) + _put("mail-mcp") + + env = Env.of_deployment(deployed) + + assert (env.id, env.stored_id, env.docker_image_artifact.id) == ("mail-mcp", SHIPPED, f"{SHIPPED}/env_image") + assert Env.of_deployment(DeployedEnv(env_id="mail-mcp", env_version=1)).stored_id == "mail-mcp" + def test_the_stores_env_of_an_aliass_id_is_the_one_it_names(aliases): _put(SHIPPED) _put("mail-mcp") From 3accad820c9e335752471bc63e52d58d58ec2207 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 23 Oct 2026 17:09:59 -0700 Subject: [PATCH 05/10] test: wait for a killed process to stop and for a cancelled run's teardown, rather than checking at once On a loaded CI host a SIGKILL lands, and an abandoned run's teardown ends, a moment after the call returns, so the local sandbox's teardown tests and the episode abandon test failed by turns. They poll for the outcome now. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../sandbox_providers/local_sandbox_test.py | 13 +++++++++++-- tst/unit/task/episode/run_test.py | 1 + 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tst/unit/providers/sandbox_providers/local_sandbox_test.py b/tst/unit/providers/sandbox_providers/local_sandbox_test.py index 6cebea5d..89ae47e8 100644 --- a/tst/unit/providers/sandbox_providers/local_sandbox_test.py +++ b/tst/unit/providers/sandbox_providers/local_sandbox_test.py @@ -947,6 +947,15 @@ def _running(pid: int) -> bool: return False +async def _stopped(pid: int, within: float = 5.0) -> bool: + """Whether ``pid`` stops running within ``within`` seconds: a kill lands a moment after it's sent, longer on a + loaded machine.""" + async with asyncio.timeout(within): + while _running(pid): + await asyncio.sleep(0.05) + return True + + async def _detached(sandbox: LocalSandbox, command: str = "sleep 300") -> int: """Start ``command`` the way a host install starts its agent, outliving the exec that started it, and wait until it reads as marked: while a process execs, /proc shows it no environment, which on a loaded machine can last.""" @@ -988,7 +997,7 @@ async def test_teardown_stops_what_a_command_left_running_and_only_that(tmp_path await _RecordingLocalSandbox(work_dir=ours.work_dir, sandbox_id=ours.sandbox_id).terminate() - assert not _running(left) + assert await _stopped(left) assert _running(other) finally: await ls._stop_marked(theirs.sandbox_id) @@ -1003,7 +1012,7 @@ async def test_teardown_kills_a_process_that_ignores_sigterm(tmp_path: Path, mon await ls._stop_marked(sandbox.sandbox_id) - assert not _running(stubborn) + assert await _stopped(stubborn) assert not ls._marked_pids(sandbox.sandbox_id) diff --git a/tst/unit/task/episode/run_test.py b/tst/unit/task/episode/run_test.py index 4cd5ccdb..8f3c4159 100644 --- a/tst/unit/task/episode/run_test.py +++ b/tst/unit/task/episode/run_test.py @@ -569,6 +569,7 @@ async def test_cancelling_the_caller_of_abandon_still_abandons_the_run(teardowns with pytest.raises(asyncio.CancelledError): await abandoning await _until(lambda: Task.get_instance(episode.id).status == "cancelled") + await _until(lambda: teardowns.finished) # the teardown ends a moment after the run is marked cancelled assert (Task.get_instance(episode.id).status,Task.get_instance(episode.id).error) == ("cancelled", "stop") assert teardowns.finished == [episode.id] From 1a257b03f480105286886da3b8b53cff399a2563 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sat, 24 Oct 2026 11:06:10 -0700 Subject: [PATCH 06/10] refactor(validate): the validator writes its env anywhere, and no store may start an env on an alias's id The hub would otherwise need its own put of the env, kept in step with the example by hand. A2AAgent.validate() now holds namespace routing and writes the mail-mcp example itself when the store has no env of the alias's id, so the hub's worker gets it too: on a remote sandbox its image is context-only and built in the VM. The alias is mail. An env store refuses to create an env under an installed env alias's id, so a deployment recorded under the alias always reloads the alias's env, and Env.of_deployment and the stored id on deployments go. Co-Authored-By: Claude Opus 5.5 (1M context) --- AGENTS.md | 16 ++-- src/agent_env/a2a_agent/a2a_agent.py | 6 +- src/agent_env/a2a_agent/validator.py | 40 ++++++++- src/agent_env/agents/AGENTS.md | 6 +- src/agent_env/cli/a2a_agent/validate.py | 34 +------- src/agent_env/env/env.py | 29 ++----- src/agent_env/env/envs/_deployment.py | 4 +- src/agent_env/env/envs/mcp_server.py | 2 +- src/agent_env/env/envs/multi_env.py | 2 +- src/agent_env/env/envs/website.py | 2 +- src/agent_env/env/store.py | 5 ++ src/agent_env/examples/mail-mcp/README.md | 2 +- src/agent_env/examples/mail-mcp/aliases.toml | 2 +- .../snapshot_utils/snapshot_series.py | 2 +- .../task_steps/apply_server_config.py | 2 +- .../task_step/task_steps/load_artifact.py | 4 +- .../verify_universe_roundtrip.py | 2 +- .../task_step/task_steps/reset_env.py | 2 +- .../task_step/task_steps/sync_env_clock.py | 2 +- .../cli/run_bundle_built_envs_test.py | 5 +- .../validator_trajectory_dag_test.py | 7 ++ tst/unit/bundle/env_toml_test.py | 7 +- tst/unit/bundle/shipped_bundles_test.py | 4 +- tst/unit/cli/a2a_agent_validate_test.py | 77 ++++++++++------- tst/unit/env/alias_get_test.py | 84 +++++++++++-------- tst/unit/store/derived_ids_test.py | 8 ++ tst/unit/store/local_id_acceptance_test.py | 8 ++ 27 files changed, 203 insertions(+), 161 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 296afb71..3888ad28 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -159,13 +159,15 @@ Out-of-tree code plugs in either through `config.toml`, as a `module:Class` poin simulator: `plan.solver_steps`), pulled in and checked as a task's reference to it would be, through per-step `step_params` overrides, so the tasks stay as written. - `aliases.toml` gives envs aliases the same way (`[envs] = ["", ...]`; the `mail-mcp` example names - its env `mail-mcp`), resolved by the planner (`Plan.env_aliases`) and by `Env.get`. The env `Env.get` gives for an - alias answers to the alias's id, so a deployment records, and steps look it up by, the id the task names; its record - also keeps the id the env is stored under (`STORED_ENV_ID_KEY`), which `Env.of_deployment` reloads it from. An alias - names its env only where namespace routing is on (the CLI), so a service needs an env of the alias's id in its store, - and only in a task's steps: an env definition (a MultiEnv's servers) naming one is refused. - `agent-env a2a-agent validate` deploys `mail-mcp` (`A2AAgent.VALIDATION_ENV_ID`), writing the example's env with - `bundle.write_bundle` (a bundle's writes and infra, no run) when the store has none of that id. + its env `mail`), resolved by the planner (`Plan.env_aliases`) and by `Env.get`, only in a task's steps: an env + definition (a MultiEnv's servers) naming one is refused. The env `Env.get` gives for an alias answers to the alias's + id, so a deployment records, and steps look it up by, the id the task names; its metadata writes go to the env the + alias names (`Env.stored_id`). An alias names its env only where namespace routing is on. An env store refuses to + create an env under an installed env alias's id (`EnvStore.put_document`), so a deployment recorded under an alias + always reloads the alias's env; an env stored under such an id before keeps taking versions. + `A2AAgent.validate()` deploys `mail` (`A2AAgent.VALIDATION_ENV_ID`) with namespace routing on, writing the example + with `bundle.write_bundle` (a bundle's writes and infra, no run) first when the store has no env of that id: on a + remote sandbox its image is context-only and built in the VM, so a service like the hub's worker needs no Docker. - Environment providers have no table: an installed package registers one as an `agent_env.env_providers` entry point named its `type`, which records carry as `env_provider_type`. - `[plugins.]` is reserved for an installed plugin's own settings, read with diff --git a/src/agent_env/a2a_agent/a2a_agent.py b/src/agent_env/a2a_agent/a2a_agent.py index 47b9b5ac..bf80dd2c 100644 --- a/src/agent_env/a2a_agent/a2a_agent.py +++ b/src/agent_env/a2a_agent/a2a_agent.py @@ -450,9 +450,9 @@ def query(cls) -> A2AAgentQuery: from .store import A2AAgentQuery, get_a2a_agent_store return A2AAgentQuery(get_a2a_agent_store()) - # The env validate gives the agent it checks: the alias of the mail-mcp example's env, which a store's env of that - # id shadows. - VALIDATION_ENV_ID: ClassVar[str] = "mail-mcp" + # The env validate gives the agent it checks: the alias of the mail-mcp example's env, which validate writes when + # the store has no env of that id. + VALIDATION_ENV_ID: ClassVar[str] = "mail" def update_metadata(self, new_metadata: dict[str, Any]) -> None: """Replace this agent's metadata in the database using compare-and-swap.""" diff --git a/src/agent_env/a2a_agent/validator.py b/src/agent_env/a2a_agent/validator.py index b814eae9..865c0fc6 100644 --- a/src/agent_env/a2a_agent/validator.py +++ b/src/agent_env/a2a_agent/validator.py @@ -18,7 +18,7 @@ import logging import uuid from dataclasses import dataclass -from typing import Callable, Optional, TYPE_CHECKING, Union +from typing import Callable, Optional, Union from agentenv_protocol.a2a_agent import ObjectChangelogApplyResponse @@ -29,7 +29,13 @@ invoke_transfer, ) from agent_env.a2a_agent.staging import transfer_store +from agent_env.a2a_agent.a2a_agent import A2AAgent +from agent_env.bundle import write_bundle from agent_env.config import get_config +from agent_env.env.store import get_env_store +from agent_env.plugins._bundles import ENV_ALIASES, installed_aliases +from agent_env.store.base import NotFoundError +from agent_env.store.routing import namespace_routing from agent_env.store.ids import derive_id, key_segment from agent_env.task_step.task_steps.a2a_agent_validator.verify_a2a_modalities import ( AUDIO_M4A_EXPECTED, @@ -56,8 +62,6 @@ ) from agent_env.task_step.task_steps.sandbox_utils.sandbox_utils import find_agent_container -if TYPE_CHECKING: - from agent_env.a2a_agent.a2a_agent import A2AAgent logger = logging.getLogger(__name__) @@ -123,6 +127,22 @@ class ModalityProbe: ] +def write_validation_env(on_progress: Callable[[str], None]) -> None: + """Write the env the validator deploys, ``A2AAgent.VALIDATION_ENV_ID``, when the store has none of that id: the env + an installed bundle gives that alias, with the infra it deploys on (``bundle.write_bundle``).""" + env_id = A2AAgent.VALIDATION_ENV_ID + try: + get_env_store().get(env_id) + return + except NotFoundError: + pass + named = installed_aliases(table=ENV_ALIASES).target(env_id) + if named is None: # deploying it says why it can't be found + return + on_progress(f"{env_id} → {named.target_id} (the alias {named.declared_by} gives it): writing it") + write_bundle(named.bundle.root, id_root=named.bundle.id_root, on_progress=on_progress) + + class A2AAgentValidator: """Static namespace for running the full A2A agent validation flow.""" @@ -135,8 +155,20 @@ async def validate( """Run the A2A agent validator task: deploy env + agent, verify card, modality probes, MCP tools, skills, snapshot extension, etc. - Returns the merged `verifications` dict from the run's task context. + Namespace routing is on throughout, so the env it deploys, ``A2AAgent.VALIDATION_ENV_ID``, can be the + ``@local`` env an installed bundle's alias names, written first (``write_validation_env``) in a thread so a + service's event loop isn't held up. Returns the merged `verifications` dict from the run's task context. """ + with namespace_routing(): + await asyncio.to_thread(write_validation_env, on_progress or (lambda line: None)) + return await A2AAgentValidator._run(agent, on_progress, litellm_api_key) + + @staticmethod + async def _run( + agent: "A2AAgent", + on_progress: Optional[Callable[[str], None]] = None, + litellm_api_key: Optional[str] = None, + ) -> dict: from agent_env.task import Task from agent_env.task_step import ( AddSkillsTaskStep, diff --git a/src/agent_env/agents/AGENTS.md b/src/agent_env/agents/AGENTS.md index fbbb859c..f13fca27 100644 --- a/src/agent_env/agents/AGENTS.md +++ b/src/agent_env/agents/AGENTS.md @@ -228,9 +228,9 @@ agent-env a2a-agent validate --id my-agent # agent-env's built agent-env a2a-agent get --id my-agent # the validation results ``` -`validate` runs a fixed set of checks against the reference env `mail-mcp`, an inbox with a `search_emails` tool: the -alias of the `mail-mcp` example's env (`src/agent_env/examples/mail-mcp/`), which `validate` writes into local stores -when the store has no env of that id, and which other stores need an env of that id for. The checks are in +`validate` runs a fixed set of checks against the reference env `mail`, an inbox with a `search_emails` tool: the alias +of the `mail-mcp` example's env (`src/agent_env/examples/mail-mcp/`), which `validate` writes before it runs, with its +image context-only on a remote sandbox, when the store has no env of that id. The checks are in `src/agent_env/a2a_agent/validator.py` and `src/agent_env/task_step/task_steps/a2a_agent_validator/`, one step each. It doesn't cover everything an agent can advertise: `triggers/v1`, `attribution-probe/v1`, host-mode `install/v1` and custom extensions need tests of your own. diff --git a/src/agent_env/cli/a2a_agent/validate.py b/src/agent_env/cli/a2a_agent/validate.py index 2968b0f4..949952c3 100644 --- a/src/agent_env/cli/a2a_agent/validate.py +++ b/src/agent_env/cli/a2a_agent/validate.py @@ -5,10 +5,8 @@ import click from agent_env.a2a_agent import A2AAgent -from agent_env.bundle import BundleError, write_bundle -from agent_env.env.bootstrap import InfraError, stores_are_local -from agent_env.env.store import get_env_store -from agent_env.plugins._bundles import ENV_ALIASES, installed_aliases +from agent_env.bundle import BundleError +from agent_env.env.bootstrap import InfraError from agent_env.store.base import NotFoundError from agent_env.utils.docker.build import DockerBuildError @@ -56,34 +54,10 @@ def validate(agent_id: str, agent_version: int | None, litellm_api_key: str | No sys.exit(1) try: - _write_validation_env() - except (BundleError, InfraError, DockerBuildError) as e: + verifications = run_validation(agent, litellm_api_key=litellm_api_key) + except (BundleError, InfraError, DockerBuildError) as e: # writing the env it deploys click.echo(click.style(f"Error: {e}", fg="red"), err=True) sys.exit(1) - verifications = run_validation(agent, litellm_api_key=litellm_api_key) if not verifications.get("a2a_agent_mcp", {}).get("passed", False): sys.exit(1) - - -def _write_validation_env() -> None: - """Write the env the validator deploys, ``A2AAgent.VALIDATION_ENV_ID``, when the store has none of that id: the env - an installed bundle gives that alias, with the infra it deploys on, into local stores, checked on the env backend - the validation deploys it on, the configured one (``--sandbox`` picks the agents'). Stores that aren't local - need an env of that id of their own, and BundleError says how to put one.""" - env_id = A2AAgent.VALIDATION_ENV_ID - try: - get_env_store().get(env_id) - return - except NotFoundError: - pass - named = installed_aliases(table=ENV_ALIASES).target(env_id) - if named is None: # deploying it says why it can't be found - return - if not stores_are_local(): - dockerfile = named.bundle.root / "envs" / named.target_id.rsplit("/", 1)[1] / "Dockerfile" - raise BundleError([f"the store has no env {env_id!r}, the one validate deploys, and validate writes " - f"{named.declared_by}'s only into local stores; put it with `agent-env env mcp-server put " - f"--id {env_id} --dockerfile {dockerfile}`"]) - click.echo(f"{env_id} → {named.target_id} (the alias {named.declared_by} gives it): writing it") - write_bundle(named.bundle.root, id_root=named.bundle.id_root, on_progress=click.echo) diff --git a/src/agent_env/env/env.py b/src/agent_env/env/env.py index fe38adfb..56dedcf6 100644 --- a/src/agent_env/env/env.py +++ b/src/agent_env/env/env.py @@ -64,17 +64,6 @@ def __init__(self, what: str, env_id: str) -> None: super().__init__(f"{what} needs a sandbox; env '{env_id}' runs outside agent-env's sandboxes") -# The deployment record of an env got by an alias names the env it was here, as the alias names it now may differ. -STORED_ENV_ID_KEY = "stored_env_id" - - -def _aliased(env: "Env", alias: str) -> "Env": - """A copy of ``env`` answering to ``alias``, which names it, while it keeps the id the store holds it under.""" - env = copy.copy(env) - env._stored_id, env.id = env.id, alias - return env - - def _holds(env_id: str) -> bool: """Whether the store has an env of ``env_id``, at any version.""" try: @@ -273,22 +262,16 @@ def get(cls, id: str, version: Optional[int] = None) -> "Env": raise NotFoundError(refusal) from None if not namespace_routing_enabled(): raise NotFoundError(f"{id!r} is an alias of {named.target_id}, which comes with {named.declared_by}, and " - f"an alias names its env only in the CLI; put an env of the id {id!r} in the store") + "an alias names its env only with namespace routing on, as the CLI and " + "A2AAgent.validate turn it on") try: env = store.get(named.target_id, version) except NotFoundError: raise NotFoundError(f"{id!r} is an alias of {named.target_id}, which comes with {named.declared_by} and " "isn't in the store yet: an `agent-env run` of a bundle that uses it writes it") from None - return _aliased(env, id) - - @classmethod - def of_deployment(cls, deployed: "DeployedEnv") -> "Env": - """The env ``deployed`` deployed, at its version: the store's env of its id, or for an env got by an alias, the - env the alias named then (``STORED_ENV_ID_KEY``), still under the alias's id, whatever the store holds now.""" - stored = (getattr(deployed, "metadata", None) or {}).get(STORED_ENV_ID_KEY) - if stored is None: - return cls.get(deployed.env_id, deployed.env_version) - return _aliased(get_env_store().get(stored, deployed.env_version), deployed.env_id) + env = copy.copy(env) + env._stored_id, env.id = env.id, id + return env @property def stored_id(self) -> str: @@ -299,7 +282,7 @@ def stored_id(self) -> str: async def from_instance_id(cls, instance_id: str) -> "Env": from .store import get_env_instance_store deployed = get_env_instance_store().get(instance_id) - env = cls.of_deployment(deployed) + env = cls.get(deployed.env_id, deployed.env_version) return await type(env).from_deployed_env(deployed) @classmethod diff --git a/src/agent_env/env/envs/_deployment.py b/src/agent_env/env/envs/_deployment.py index 70b89c3a..2d4db4d7 100644 --- a/src/agent_env/env/envs/_deployment.py +++ b/src/agent_env/env/envs/_deployment.py @@ -10,7 +10,7 @@ from agentenv_protocol import METHOD_ADD, METHOD_RESET, RPC_PATH, FilePart, client as protocol_v1 from agent_env.env import legacy_protocol -from agent_env.env.env import STORED_ENV_ID_KEY, DeployedSandboxEnv, gateway_url_of +from agent_env.env.env import DeployedSandboxEnv, gateway_url_of from agent_env.env.store import register_env_instance from agent_env.store.base import ObjectNotFoundError @@ -66,8 +66,6 @@ async def deploy_through_provider(env: Env, *, environment_name: str | None, ttl else: check_plugin_record(env, deployed) env._gateway_url = gateway_url_of(deployed) - if env.stored_id != env.id: # got by an alias: the record keeps the env it was, for a reload - deployed.metadata = {**(deployed.metadata or {}), STORED_ENV_ID_KEY: env.stored_id} deployed = register_env_instance(deployed, ttl_seconds) env._instance_id = deployed.instance_id env._deployed, env._replaced = deployed, replaced diff --git a/src/agent_env/env/envs/mcp_server.py b/src/agent_env/env/envs/mcp_server.py index ea345ba5..46128d48 100644 --- a/src/agent_env/env/envs/mcp_server.py +++ b/src/agent_env/env/envs/mcp_server.py @@ -118,7 +118,7 @@ def deploy_refusal(self, **options) -> str | None: @classmethod async def from_deployed_env(cls, deployed: DeployedEnv) -> MCPServerEnv: - env = Env.of_deployment(deployed) + env = Env.get(deployed.env_id, deployed.env_version) if not isinstance(env, MCPServerEnv): raise TypeError(f"Expected MCPServerEnv, got {type(env).__name__}") if env._env_provider is None: diff --git a/src/agent_env/env/envs/multi_env.py b/src/agent_env/env/envs/multi_env.py index f215da13..ec163dc5 100644 --- a/src/agent_env/env/envs/multi_env.py +++ b/src/agent_env/env/envs/multi_env.py @@ -151,7 +151,7 @@ def deploy_refusal(self, **options) -> str | None: @classmethod async def from_deployed_env(cls, deployed: DeployedEnv) -> MultiEnv: - env = Env.of_deployment(deployed) + env = Env.get(deployed.env_id, deployed.env_version) if not isinstance(env, MultiEnv): raise TypeError(f"Expected MultiEnv, got {type(env).__name__}") provider = builtin_provider_for(env.env_provider_type) or plugin_provider_like_a_builtin(env.env_provider_type) diff --git a/src/agent_env/env/envs/website.py b/src/agent_env/env/envs/website.py index ceb24080..a35dd30e 100644 --- a/src/agent_env/env/envs/website.py +++ b/src/agent_env/env/envs/website.py @@ -138,7 +138,7 @@ def deploy_refusal(self, **options) -> str | None: @classmethod async def from_deployed_env(cls, deployed: DeployedEnv) -> WebsiteEnv: - env = Env.of_deployment(deployed) + env = Env.get(deployed.env_id, deployed.env_version) if not isinstance(env, WebsiteEnv): raise TypeError(f"Expected WebsiteEnv, got {type(env).__name__}") if env._env_provider is None: diff --git a/src/agent_env/env/store.py b/src/agent_env/env/store.py index dcf5f276..ccdf265f 100644 --- a/src/agent_env/env/store.py +++ b/src/agent_env/env/store.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Optional, Self, TypeVar from agent_env.plugins import _registration +from agent_env.plugins._bundles import ENV_ALIASES, installed_aliases from agent_env.store.base import ConcurrentModificationError, NotFoundError from agent_env.config import get_config from agent_env.store.document_store import ( @@ -100,6 +101,10 @@ def put_document(self, env: _E) -> _E: ) if existing and existing.get("type") != env.type: raise ValueError(f"Env id '{env.id}' already exists as type '{existing.get('type')}'; cannot put as type '{env.type}'.") + # A deployment recorded under an alias reloads the alias's env, so no env of the store's own starts on its id. + if not existing and (named := installed_aliases(table=ENV_ALIASES).target(env.id)) is not None: + raise ValueError(f"env id {env.id!r} is an alias of {named.target_id}, which comes with " + f"{named.declared_by}, and a task naming it deploys that env; give this env another id") env.version = self._versioned_cache.for_store(store).put(env) return env diff --git a/src/agent_env/examples/mail-mcp/README.md b/src/agent_env/examples/mail-mcp/README.md index fb43ad3c..01746b0d 100644 --- a/src/agent_env/examples/mail-mcp/README.md +++ b/src/agent_env/examples/mail-mcp/README.md @@ -3,7 +3,7 @@ An MCP server with an inbox to search, deployed through the gateway: the env the `agent-env a2a-agent validate` deploys it, by its alias, beside the agent it checks. It needs Docker. `envs/mail/` is the server: a `Dockerfile` and a `server.py` whose one tool, `search_emails`, returns the emails holding every word of its query from a fixed inbox of three, named `mail` by its environment card. `aliases.toml` gives it the alias -`mail-mcp`, so any task can name it `mail-mcp`. `tasks/smoke.json` deploys it and checks, with +`mail`, so any task can name it `mail`. `tasks/smoke.json` deploys it and checks, with `artifacts/check_mail/verify.py`, that a search for "invoice" finds the inbox's one invoice. The first run also builds the gateway and service database it deploys on, which takes minutes. diff --git a/src/agent_env/examples/mail-mcp/aliases.toml b/src/agent_env/examples/mail-mcp/aliases.toml index 9fddc0ef..9600e278 100644 --- a/src/agent_env/examples/mail-mcp/aliases.toml +++ b/src/agent_env/examples/mail-mcp/aliases.toml @@ -1,3 +1,3 @@ # Ids a task can name this bundle's envs by; a store's env of the id wins over an alias. [envs] -mail = ["mail-mcp"] +mail = ["mail"] diff --git a/src/agent_env/task_step/snapshot_utils/snapshot_series.py b/src/agent_env/task_step/snapshot_utils/snapshot_series.py index 780340ee..f0b8817e 100644 --- a/src/agent_env/task_step/snapshot_utils/snapshot_series.py +++ b/src/agent_env/task_step/snapshot_utils/snapshot_series.py @@ -502,7 +502,7 @@ async def _capture_env_state( try: # Pin the env version the sandbox actually runs, so service # enumeration can't drift if the env was re-registered since deploy. - env = await asyncio.to_thread(Env.of_deployment, deployed) + env = await asyncio.to_thread(Env.get, deployed.env_id, deployed.env_version) result = await SnapshotEnvTaskStep.snapshot_env_state( env=env, gateway_url=deployed.gateway_url, diff --git a/src/agent_env/task_step/task_steps/apply_server_config.py b/src/agent_env/task_step/task_steps/apply_server_config.py index 598be40e..d1495fb9 100644 --- a/src/agent_env/task_step/task_steps/apply_server_config.py +++ b/src/agent_env/task_step/task_steps/apply_server_config.py @@ -143,7 +143,7 @@ def _expand_directives(self, deployed) -> list[ConfigDirective]: from agent_env.env.env import Env - env = Env.of_deployment(deployed) + env = Env.get(deployed.env_id, deployed.env_version) environment_names = [ e.environment_name for e in (getattr(env, "mcp_server_envs", None) or []) ] diff --git a/src/agent_env/task_step/task_steps/load_artifact.py b/src/agent_env/task_step/task_steps/load_artifact.py index b42458ab..1555c9ae 100644 --- a/src/agent_env/task_step/task_steps/load_artifact.py +++ b/src/agent_env/task_step/task_steps/load_artifact.py @@ -499,7 +499,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: # Only a FileArtifactUniverse or a FileArtifact reaches here — `stage_as_files` # excludes a service universe whenever an env is targeted. if env is None: - env_obj = Env.of_deployment(deployed) + env_obj = Env.get(deployed.env_id, deployed.env_version) env = await type(env_obj).from_deployed_env(deployed) result = await env.load_file_artifact_universe(artifact, destination_path=destination_path) context.metadata.setdefault("loaded_file_artifact_universes", []).append({ @@ -583,7 +583,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: if deployed is None: raise RuntimeError(f"`env_id` is required when loading a {artifact.type}") if env is None: - env_obj = Env.of_deployment(deployed) + env_obj = Env.get(deployed.env_id, deployed.env_version) env = await type(env_obj).from_deployed_env(deployed) if isinstance(artifact, EnvironmentArtifact): await env.load_environment_artifact(artifact) diff --git a/src/agent_env/task_step/task_steps/multienv_validator/verify_universe_roundtrip.py b/src/agent_env/task_step/task_steps/multienv_validator/verify_universe_roundtrip.py index f5f0257a..3e5b6b3d 100644 --- a/src/agent_env/task_step/task_steps/multienv_validator/verify_universe_roundtrip.py +++ b/src/agent_env/task_step/task_steps/multienv_validator/verify_universe_roundtrip.py @@ -102,7 +102,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: deployed = next((d for d in context.deployed_envs if d.env_id == self.env_id), None) if deployed is None: raise RuntimeError(f"Env '{self.env_id}' not found in context.deployed_envs") - env = Env.of_deployment(deployed) + env = Env.get(deployed.env_id, deployed.env_version) env = await type(env).from_deployed_env(deployed) universe = EnvironmentUniverseArtifact.get(self.universe_artifact_id, self.universe_artifact_version) environment_artifacts = universe.get_environment_artifacts() diff --git a/src/agent_env/task_step/task_steps/reset_env.py b/src/agent_env/task_step/task_steps/reset_env.py index fee656d7..88059fdd 100644 --- a/src/agent_env/task_step/task_steps/reset_env.py +++ b/src/agent_env/task_step/task_steps/reset_env.py @@ -58,7 +58,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: logger.warning("reset_env: no deployed env '%s' in context; skipping", self.env_id) return context try: - env = Env.of_deployment(deployed) + env = Env.get(deployed.env_id, deployed.env_version) await env.reset(deployed) logger.info("reset_env: reset env '%s'", self.env_id) except NotImplementedError: diff --git a/src/agent_env/task_step/task_steps/sync_env_clock.py b/src/agent_env/task_step/task_steps/sync_env_clock.py index 41ba3707..1dc48a55 100644 --- a/src/agent_env/task_step/task_steps/sync_env_clock.py +++ b/src/agent_env/task_step/task_steps/sync_env_clock.py @@ -72,7 +72,7 @@ def from_dict(cls, data: dict) -> SyncEnvClockTaskStep: def _service_names(self, deployed) -> list[str]: """Backing MCP server names for the env (mirrors apply_server_config's enumeration).""" from agent_env.env.env import Env - env = Env.of_deployment(deployed) + env = Env.get(deployed.env_id, deployed.env_version) names = [e.environment_name for e in (getattr(env, "mcp_server_envs", None) or [])] if not names and getattr(env, "environment_name", None): names = [env.environment_name] diff --git a/tst/integration/cli/run_bundle_built_envs_test.py b/tst/integration/cli/run_bundle_built_envs_test.py index 8aa43698..386cfcd4 100644 --- a/tst/integration/cli/run_bundle_built_envs_test.py +++ b/tst/integration/cli/run_bundle_built_envs_test.py @@ -572,7 +572,7 @@ def test_the_shipped_mail_mcp_example_runs_and_a_task_outside_a_bundle_deploys_i assert first.exit_code == 0, first.output assert "tasks/smoke.json v1: passed (finds-the-invoice: 1)" in first.output, first.output - (state / "by-alias.json").write_text(json.dumps([{"id": "env", "type": "deploy_env", "env_id": "mail-mcp"}])) + (state / "by-alias.json").write_text(json.dumps([{"id": "env", "type": "deploy_env", "env_id": "mail"}])) created = CliRunner().invoke(cli, ["task", "create", "--id", "by-alias", str(state / "by-alias.json")]) ran = CliRunner().invoke(cli, ["task", "run", "--id", "by-alias", "--output-dir", str(state / "out")]) @@ -580,8 +580,7 @@ def test_the_shipped_mail_mcp_example_runs_and_a_task_outside_a_bundle_deploys_i assert ran.exit_code == 0, ran.output [written] = (state / "out").iterdir() [deployed] = json.loads(written.read_text())["deployed_envs"] - assert deployed["env_id"] == "mail-mcp" and deployed["gateway_url"], deployed - assert deployed["metadata"]["stored_env_id"] == "@local/agentenv-framework/mail-mcp/mail", deployed + assert deployed["env_id"] == "mail" and deployed["gateway_url"], deployed ECHO = REPO / "src" / "agent_env" / "examples" / "echo-mcp" diff --git a/tst/unit/a2a_agent/validator_trajectory_dag_test.py b/tst/unit/a2a_agent/validator_trajectory_dag_test.py index 32298869..26dd0ea4 100644 --- a/tst/unit/a2a_agent/validator_trajectory_dag_test.py +++ b/tst/unit/a2a_agent/validator_trajectory_dag_test.py @@ -18,6 +18,7 @@ from agentenv_protocol.transfers import HttpPutGrant from agent_env.a2a_agent import A2AAgent, object_transfer +from agent_env.a2a_agent import validator as validator_module from agent_env.a2a_agent.validator import A2AAgentValidator from agent_env.config import Config, configure from agent_env.task import Task @@ -32,6 +33,12 @@ ) +@pytest.fixture(autouse=True) +def no_validation_env_write(monkeypatch): + """The env the validator writes first is a2a_agent_validate_test's; these runs write none.""" + monkeypatch.setattr(validator_module, "write_validation_env", lambda on_progress: None) + + def _context_with_trajectory_agent( mcp_verification: dict | None, *, get_method: dict | None = None ) -> TaskStepContext: diff --git a/tst/unit/bundle/env_toml_test.py b/tst/unit/bundle/env_toml_test.py index 5d9b881f..f904b635 100644 --- a/tst/unit/bundle/env_toml_test.py +++ b/tst/unit/bundle/env_toml_test.py @@ -145,18 +145,19 @@ def test_a_website_builds_dockerfile_backend_and_dockerfile_frontend_and_takes_i def test_a_multi_names_bundle_and_store_envs_and_is_rewritten_when_a_child_is(bundle_dir, builds): with namespace_routing(): - MCPServerEnv.put(id="mail", docker_image_artifact=_store_image("mail-image"), environment_name="mail") + MCPServerEnv.put(id="inbox", docker_image_artifact=_store_image("inbox-image"), environment_name="inbox") _bundle(bundle_dir, { **_mcp("crm"), "envs/shop/Dockerfile.backend": DOCKERFILE, "envs/shop/Dockerfile.frontend": DOCKERFILE, "envs/shop/env.toml": 'type = "website"\nenvironment_name = "shop"\n', - "envs/suite/env.toml": 'type = "multi"\nmcp_server_envs = ["crm", "mail"]\nwebsite_envs = ["shop"]\nname = "suite"\n', + "envs/suite/env.toml": ('type = "multi"\nmcp_server_envs = ["crm", "inbox"]\nwebsite_envs = ["shop"]\n' + 'name = "suite"\n'), }, "suite") _run(bundle_dir) suite = Env.get(f"{ROOT}/suite") assert isinstance(suite, MultiEnv) and suite.name == "suite" assert [(env.id, env.version) for env in [*suite.mcp_server_envs, *suite.website_envs]] == [ - (f"{ROOT}/crm", 1), ("mail", 1), (f"{ROOT}/shop", 1)] + (f"{ROOT}/crm", 1), ("inbox", 1), (f"{ROOT}/shop", 1)] (bundle_dir / "envs/crm/server.py").write_text(CARD.format("crm") + "# edited\n") rewritten = _summary(_run(bundle_dir)) diff --git a/tst/unit/bundle/shipped_bundles_test.py b/tst/unit/bundle/shipped_bundles_test.py index 5a992506..f918f524 100644 --- a/tst/unit/bundle/shipped_bundles_test.py +++ b/tst/unit/bundle/shipped_bundles_test.py @@ -55,8 +55,8 @@ def test_the_shipped_env_alias_names_the_mail_mcp_examples_env_which_the_validat assert table.refused == {} and table.refused_claims == {} assert {alias: named.target_id for alias, named in table.aliases.items()} == { - "mail-mcp": "@local/agentenv-framework/mail-mcp/mail"} - assert A2AAgent.VALIDATION_ENV_ID == "mail-mcp" + "mail": "@local/agentenv-framework/mail-mcp/mail"} + assert A2AAgent.VALIDATION_ENV_ID == "mail" def test_a_task_naming_no_agent_plans_with_the_shipped_default(tmp_path, monkeypatch, local_stores, cli_routing): monkeypatch.setenv("HOME", str(tmp_path)) diff --git a/tst/unit/cli/a2a_agent_validate_test.py b/tst/unit/cli/a2a_agent_validate_test.py index 72c0bc3c..a4433de1 100644 --- a/tst/unit/cli/a2a_agent_validate_test.py +++ b/tst/unit/cli/a2a_agent_validate_test.py @@ -1,5 +1,5 @@ -"""`agent-env a2a-agent validate` writes the env it deploys, the mail-mcp example's by its alias, when the store has -none of that id.""" +"""The A2A agent validator deploys the mail-mcp example's env by its alias, `mail`, writing it first when the store has +no env of that id, with namespace routing on throughout so the alias's @local env resolves, in the CLI or a service.""" import importlib @@ -7,67 +7,82 @@ from click.testing import CliRunner from agent_env.a2a_agent import A2AAgent +from agent_env.a2a_agent import validator as validator_module from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.store import get_artifact_store from agent_env.bundle import BundleError from agent_env.cli import cli +from agent_env.env import store as env_store_module from agent_env.env.envs.mcp_server import MCPServerEnv +from agent_env.plugins._bundles import InstalledAliases +from agent_env.store.routing import namespace_routing_enabled # By its path: the package exports the command under the module's name. validate_module = importlib.import_module("agent_env.cli.a2a_agent.validate") @pytest.fixture -def written(monkeypatch, local_stores, cli_routing): - """The bundles validate writes, as (root, id_root, the sandbox each is checked on).""" +def written(monkeypatch, local_stores): + """The bundles the validator writes, as (folder, id_root).""" calls = [] - monkeypatch.setattr(validate_module, "write_bundle", lambda root, **kwargs: calls.append( - (root.name, kwargs["id_root"], kwargs.get("sandbox")))) + monkeypatch.setattr(validator_module, "write_bundle", + lambda root, **kwargs: calls.append((root.name, kwargs["id_root"]))) return calls def test_the_validator_deploys_the_mail_mcp_examples_env_by_its_alias(): - assert A2AAgent.VALIDATION_ENV_ID == "mail-mcp" + assert A2AAgent.VALIDATION_ENV_ID == "mail" -def test_validate_writes_the_example_its_alias_names_on_the_env_backend_when_the_store_has_no_env_of_that_id(written): - """Checked where the validation deploys the env, the configured env backend, not on --sandbox, the agents'.""" - validate_module._write_validation_env() +def test_the_example_its_alias_names_is_written_when_the_store_has_no_env_of_that_id(written): + lines = [] - assert written == [("mail-mcp", "@local/agentenv-framework/mail-mcp", None)] + validator_module.write_validation_env(lines.append) + assert written == [("mail-mcp", "@local/agentenv-framework/mail-mcp")] + assert lines == ["mail → @local/agentenv-framework/mail-mcp/mail (the alias agentenv-framework's mail-mcp bundle " + "gives it): writing it"] -def test_the_stores_env_of_that_id_is_the_one_validate_deploys(written): + +def test_an_env_the_store_held_under_that_id_before_is_the_one_deployed(written, monkeypatch): image = get_artifact_store().put_document(DockerImageArtifact( - id="mail-mcp/env_image", description="d", image_name="ghcr.io/acme/mail:1", tar_gz_s3_url="s3://b/x")) - MCPServerEnv.put(id="mail-mcp", docker_image_artifact=image, environment_name="mail") + id="mail/env_image", description="d", image_name="ghcr.io/acme/mail:1", tar_gz_s3_url="s3://b/x")) + with monkeypatch.context() as before_the_alias: + before_the_alias.setattr(env_store_module, "installed_aliases", + lambda table: InstalledAliases({}, {}, {}, table)) + MCPServerEnv.put(id="mail", docker_image_artifact=image, environment_name="mail") - validate_module._write_validation_env() + validator_module.write_validation_env(lambda line: None) assert written == [] -def test_stores_that_arent_local_need_an_env_of_that_id_and_are_told_how_to_put_one(written, monkeypatch): - monkeypatch.setattr(validate_module, "stores_are_local", lambda: False) +@pytest.mark.asyncio +async def test_validation_writes_the_env_then_runs_with_namespace_routing_on_throughout(monkeypatch): + seen = [] + monkeypatch.setattr(validator_module, "write_validation_env", + lambda on_progress: seen.append(("write", namespace_routing_enabled()))) - with pytest.raises(BundleError, match="the store has no env 'mail-mcp', the one validate deploys, and validate " - "writes agentenv-framework's mail-mcp bundle's only into local stores; put " - r"it with `agent-env env mcp-server put --id mail-mcp --dockerfile .*" - r"mail-mcp/envs/mail/Dockerfile`"): - validate_module._write_validation_env() - assert written == [] + async def run(agent, on_progress, litellm_api_key): + seen.append(("run", namespace_routing_enabled())) + return {"done": True} + + monkeypatch.setattr(validator_module.A2AAgentValidator, "_run", staticmethod(run)) + assert await validator_module.A2AAgentValidator.validate(object()) == {"done": True} + assert seen == [("write", True), ("run", True)] and not namespace_routing_enabled() -def test_the_cli_writes_the_env_before_it_validates(written, monkeypatch): + +def test_the_cli_says_why_the_env_couldnt_be_written(monkeypatch, local_stores, cli_routing): image = get_artifact_store().put_document(DockerImageArtifact( id="agent-image", description="d", image_name="ghcr.io/acme/agent:1", tar_gz_s3_url="s3://b/x")) A2AAgent.put(id="mine", docker_image_artifact=image) - order = [] - monkeypatch.setattr(validate_module, "_write_validation_env", lambda: order.append("write")) - monkeypatch.setattr(validate_module, "run_validation", lambda agent, litellm_api_key=None: order.append( - "validate") or {"a2a_agent_mcp": {"passed": True}}) - result = CliRunner().invoke(cli, ["a2a-agent", "validate", "--id", "mine", "--sandbox", "local"]) + def refuse(agent, litellm_api_key=None): + raise BundleError(["tasks/smoke.json: no docker here"]) + + monkeypatch.setattr(validate_module, "run_validation", refuse) + + result = CliRunner().invoke(cli, ["a2a-agent", "validate", "--id", "mine"]) - assert result.exit_code == 0, result.output - assert order == ["write", "validate"] + assert result.exit_code == 1 and "Error: tasks/smoke.json: no docker here" in result.output diff --git a/tst/unit/env/alias_get_test.py b/tst/unit/env/alias_get_test.py index 7c20294e..5dc1e1c5 100644 --- a/tst/unit/env/alias_get_test.py +++ b/tst/unit/env/alias_get_test.py @@ -6,9 +6,10 @@ from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.store import get_artifact_store from agent_env.env import env as env_module -from agent_env.env.env import STORED_ENV_ID_KEY, DeployedEnv, Env +from agent_env.env import store as env_store_module +from agent_env.env.env import Env from agent_env.env.envs.mcp_server import MCPServerEnv -from agent_env.plugins._bundles import ALIASES_FILE, InstalledBundle, installed_aliases +from agent_env.plugins._bundles import ALIASES_FILE, InstalledAliases, InstalledBundle, installed_aliases from agent_env.store.base import NotFoundError from agent_env.store.routing import namespace_routing @@ -17,16 +18,18 @@ @pytest.fixture def bundles(tmp_path, monkeypatch, local_stores): - """agent-env's mail-mcp bundle gives its env the alias mail-mcp, and two plugin bundles both give helper.""" + """agent-env's mail-mcp bundle gives its env the alias mail, and two plugin bundles both give helper; the env store + sees them too.""" installed = [] - for package, name, toml in [("agentenv-framework", "mail-mcp", '[envs]\nmail = ["mail-mcp"]\n'), + for package, name, toml in [("agentenv-framework", "mail-mcp", '[envs]\nmail = ["mail"]\n'), ("acme-kit", "kit", '[envs]\na = ["helper"]\n'), ("zeta-tools", "tools", '[envs]\nb = ["helper"]\n')]: root = tmp_path / package / name root.mkdir(parents=True) (root / ALIASES_FILE).write_text(toml) installed.append(InstalledBundle(name, package, "1.0", package, root, None, None)) - monkeypatch.setattr(env_module, "installed_aliases", lambda table: installed_aliases(installed, table)) + for module in (env_module, env_store_module): + monkeypatch.setattr(module, "installed_aliases", lambda table: installed_aliases(installed, table)) @pytest.fixture @@ -40,60 +43,55 @@ def _put(env_id): return MCPServerEnv.put(id=env_id, docker_image_artifact=image, environment_name="mail") +def _put_before_the_alias(monkeypatch, env_id): + """An env put under ``env_id`` while no installed bundle gave that alias.""" + with monkeypatch.context() as before: + before.setattr(env_store_module, "installed_aliases", lambda table: InstalledAliases({}, {}, {}, table)) + return _put(env_id) + + def test_an_alias_names_the_env_its_bundle_gives_it_under_the_aliass_id(aliases): _put(SHIPPED) _put(SHIPPED) - env = Env.get("mail-mcp") + env = Env.get("mail") - assert (env.id, env.version, env.environment_name) == ("mail-mcp", 2, "mail") - assert Env.get("mail-mcp", 1).version == 1 # a deployment's record reloads the version it deployed + assert (env.id, env.version, env.environment_name) == ("mail", 2, "mail") + assert Env.get("mail", 1).version == 1 # a deployment's record reloads the version it deployed assert Env.get(SHIPPED).id == SHIPPED - def test_an_env_got_by_an_alias_writes_its_metadata_to_the_env_the_alias_names(aliases): _put(SHIPPED) - env = Env.get("mail-mcp") + env = Env.get("mail") env.update_metadata({**env.metadata, "checked": "yes"}) env.merge_metadata({"merged": "too"}) - assert (env.id, env.stored_id) == ("mail-mcp", SHIPPED) + assert (env.id, env.stored_id) == ("mail", SHIPPED) assert {key: Env.get(SHIPPED).metadata[key] for key in ("checked", "merged")} == {"checked": "yes", "merged": "too"} -def test_a_deployment_of_an_env_got_by_an_alias_reloads_that_env_whatever_the_store_holds_since(aliases): - """A record of the alias's env names the env it was, so one the store gains under the alias's id later isn't it.""" - _put(SHIPPED) - deployed = DeployedEnv(env_id="mail-mcp", env_version=1, metadata={STORED_ENV_ID_KEY: SHIPPED}) - _put("mail-mcp") - - env = Env.of_deployment(deployed) - - assert (env.id, env.stored_id, env.docker_image_artifact.id) == ("mail-mcp", SHIPPED, f"{SHIPPED}/env_image") - assert Env.of_deployment(DeployedEnv(env_id="mail-mcp", env_version=1)).stored_id == "mail-mcp" - -def test_the_stores_env_of_an_aliass_id_is_the_one_it_names(aliases): +def test_an_env_the_store_held_under_an_aliass_id_before_is_the_one_it_names(aliases, monkeypatch): _put(SHIPPED) - _put("mail-mcp") + _put_before_the_alias(monkeypatch, "mail") - assert Env.get("mail-mcp").docker_image_artifact.id == "mail-mcp/env_image" + assert Env.get("mail").docker_image_artifact.id == "mail/env_image" -def test_a_version_the_stores_env_of_an_alias_id_lacks_is_not_found(aliases): +def test_a_version_the_stores_env_of_an_alias_id_lacks_is_not_found(aliases, monkeypatch): _put(SHIPPED) _put(SHIPPED) - _put("mail-mcp") + _put_before_the_alias(monkeypatch, "mail") with pytest.raises(NotFoundError): - Env.get("mail-mcp", 2) + Env.get("mail", 2) def test_an_alias_whose_env_no_run_has_written_says_so(aliases): - with pytest.raises(NotFoundError, match=f"'mail-mcp' is an alias of {SHIPPED}, which comes with " + with pytest.raises(NotFoundError, match=f"'mail' is an alias of {SHIPPED}, which comes with " "agentenv-framework's mail-mcp bundle and isn't in the store yet"): - Env.get("mail-mcp") + Env.get("mail") def test_an_alias_two_bundles_give_is_refused_naming_both(aliases): @@ -103,15 +101,27 @@ def test_an_alias_two_bundles_give_is_refused_naming_both(aliases): def test_an_id_close_to_an_alias_points_at_it(aliases): - with pytest.raises(NotFoundError, match="there is no env 'mail_mcp' in the store; did you mean the alias " - "'mail-mcp'"): - Env.get("mail_mcp") + with pytest.raises(NotFoundError, match="there is no env 'mai' in the store; did you mean the alias 'mail'"): + Env.get("mai") -def test_without_namespace_routing_an_alias_names_no_env_and_says_to_put_one(bundles): +def test_without_namespace_routing_an_alias_names_no_env_and_says_so(bundles): with namespace_routing(): _put(SHIPPED) - with pytest.raises(NotFoundError, match="an alias names its env only in the CLI; put an env of the id " - "'mail-mcp' in the store"): - Env.get("mail-mcp") + with pytest.raises(NotFoundError, match="an alias names its env only with namespace routing on"): + Env.get("mail") + + +def test_the_store_refuses_to_start_an_env_under_an_installed_aliass_id(aliases): + """A deployment recorded under the alias would otherwise reload that env in place of the one deployed.""" + with pytest.raises(ValueError, match=f"env id 'mail' is an alias of {SHIPPED}, which comes with " + "agentenv-framework's mail-mcp bundle, and a task naming it deploys that env; " + "give this env another id"): + _put("mail") + + +def test_an_env_the_store_held_under_an_aliass_id_before_keeps_taking_versions(aliases, monkeypatch): + _put_before_the_alias(monkeypatch, "mail") + + assert _put("mail").version == 2 diff --git a/tst/unit/store/derived_ids_test.py b/tst/unit/store/derived_ids_test.py index eada5279..20c6c97e 100644 --- a/tst/unit/store/derived_ids_test.py +++ b/tst/unit/store/derived_ids_test.py @@ -15,6 +15,7 @@ import agent_env.providers.sandbox_providers.sandbox_provider as sandbox_provider from agent_env.a2a_agent.a2a_agent import A2AAgent +from agent_env.a2a_agent import validator as validator_module from agent_env.a2a_agent.validator import A2AAgentValidator from agent_env.artifact import DockerImageArtifact, FileArtifactUniverse from agent_env.cli import cli @@ -34,6 +35,13 @@ from agent_env.task_step.task_steps.verifiers import run_container_unit_tests_verifier as verifier_module from tst.unit.store.fakes import CollectingVm, SigningObjectStore, reattach_for_snapshot + +@pytest.fixture(autouse=True) +def no_validation_env_write(monkeypatch): + """The env the validator writes first is a2a_agent_validate_test's; these runs write none.""" + monkeypatch.setattr(validator_module, "write_validation_env", lambda on_progress: None) + + LOCAL = "@local/~/work/triage" LOCAL_ENV = f"{LOCAL}/envs/tickets" LOCAL_UNIVERSE = f"{LOCAL}/artifacts/seed" diff --git a/tst/unit/store/local_id_acceptance_test.py b/tst/unit/store/local_id_acceptance_test.py index df9d5412..49b6e602 100644 --- a/tst/unit/store/local_id_acceptance_test.py +++ b/tst/unit/store/local_id_acceptance_test.py @@ -10,6 +10,7 @@ from click.testing import CliRunner from agent_env.a2a_agent.a2a_agent import A2AAgent +from agent_env.a2a_agent import validator as validator_module from agent_env.a2a_agent.validator import A2AAgentValidator from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.artifacts.file import FileArtifact @@ -33,6 +34,13 @@ from agent_env.task import Task from tst.unit.store.fakes import FakeDocumentStore, FakeImageStore, SigningObjectStore, reattach_for_snapshot + +@pytest.fixture(autouse=True) +def no_validation_env_write(monkeypatch): + """The env the validator writes first is a2a_agent_validate_test's; these runs write none.""" + monkeypatch.setattr(validator_module, "write_validation_env", lambda on_progress: None) + + LOCAL_ID = "@local/t/Chaos" LOCAL_ENV = "@local/~/bundle/envs/e" UNIVERSE = "@local/~/bundle/artifacts/data" From b690ab58f33bc799e52ea245b79d5ca2f7db985b Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sat, 24 Oct 2026 11:47:13 -0700 Subject: [PATCH 07/10] test: name the gateway test's store env inbox, since mail is now an env alias's id Co-Authored-By: Claude Opus 5.5 (1M context) --- .../cli/run_bundle_built_envs_test.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tst/integration/cli/run_bundle_built_envs_test.py b/tst/integration/cli/run_bundle_built_envs_test.py index 386cfcd4..a2d86210 100644 --- a/tst/integration/cli/run_bundle_built_envs_test.py +++ b/tst/integration/cli/run_bundle_built_envs_test.py @@ -215,7 +215,7 @@ def _gateway_bundle(root): """An env of each kind a bundle writes, all deployed through the gateway: items and stock are the items server, built from its folder and named by its card, and built from docker/Dockerfile under a name of its own; depot is the items server built from an image folder, artifacts/depot-image; relay is an MCP server over the store image - the CLI put for mail; shop is a website; suite is a multi of items, mail and shop.""" + the CLI put for inbox; shop is a website; suite is a multi of items, inbox and shop.""" envs = root / "envs" _items(envs / "items", "v1") _items(envs / "stock", "v1", dockerfile="docker/Dockerfile") @@ -224,11 +224,11 @@ def _gateway_bundle(root): (envs / "depot").mkdir() (envs / "depot/env.toml").write_text('image = "depot-image"\nenvironment_name = "depot"\n') (envs / "relay").mkdir() - (envs / "relay/env.toml").write_text('image = "mail__env_image"\nenvironment_name = "relay"\n') + (envs / "relay/env.toml").write_text('image = "inbox__env_image"\nenvironment_name = "relay"\n') _shop(envs / "shop") (envs / "suite").mkdir() (envs / "suite/env.toml").write_text( - 'type = "multi"\nmcp_server_envs = ["items", "mail"]\nwebsite_envs = ["shop"]\nname = "suite"\n') + 'type = "multi"\nmcp_server_envs = ["items", "inbox"]\nwebsite_envs = ["shop"]\nname = "suite"\n') _check(root, "items", items=["items"]) _check(root, "stock", items=["stock"]) _check(root, "depot", items=["depot"]) @@ -238,9 +238,9 @@ def _gateway_bundle(root): return root -def _put_mail(): - """The slack server as a store env, mail, put by the CLI as a user would before a bundle names it.""" - put = CliRunner().invoke(cli, ["env", "mcp-server", "put", "--id", "mail", "--environment-name", "mail", +def _put_inbox(): + """The slack server as a store env, inbox, put by the CLI as a user would before a bundle names it.""" + put = CliRunner().invoke(cli, ["env", "mcp-server", "put", "--id", "inbox", "--environment-name", "mail", "--dockerfile", str(DATA / "slack_mcp/Dockerfile"), "--context", str(DATA), "--platform", ""]) assert put.exit_code == 0, put.output @@ -253,7 +253,7 @@ def _put_mail(): def test_each_kind_of_bundle_env_deploys_through_the_gateway_and_a_child_edit_rewrites_its_multi(state): - _put_mail() + _put_inbox() root = _gateway_bundle(state / "gateway-bundle") first = _run(root) @@ -267,12 +267,12 @@ def test_each_kind_of_bundle_env_deploys_through_the_gateway_and_a_child_edit_re with namespace_routing(): ids = {entry.name: entry.id for entry in parse_bundle(root).entries} written = {name: Env.get(ids[name]) for name in GATEWAY_ENVS} - mail = Env.get("mail") + inbox = Env.get("inbox") assert {name: env.environment_name for name, env in written.items() if name != "suite"} == { "items": "items", "stock": "stock", "depot": "depot", "relay": "relay", "shop": "shop"} assert written["depot"].docker_image_artifact.id == ids["depot-image"] - assert written["relay"].docker_image_artifact.id == mail.docker_image_artifact.id == "mail__env_image" - assert [child.id for child in written["suite"].mcp_server_envs] == [ids["items"], "mail"] + assert written["relay"].docker_image_artifact.id == inbox.docker_image_artifact.id == "inbox__env_image" + assert [child.id for child in written["suite"].mcp_server_envs] == [ids["items"], "inbox"] assert [child.id for child in written["suite"].website_envs] == [ids["shop"]] assert written["suite"].name == "suite" From 31e98e3f4d930c059d153e413af609a23efc4e0a Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sat, 24 Oct 2026 12:21:05 -0700 Subject: [PATCH 08/10] test: treat a process reaped while its /proc stat is read as gone Co-Authored-By: Claude Opus 5.5 (1M context) --- tst/unit/agents/gemini/test_gemini_bridge_cancel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tst/unit/agents/gemini/test_gemini_bridge_cancel.py b/tst/unit/agents/gemini/test_gemini_bridge_cancel.py index 38529ec6..85a11be2 100644 --- a/tst/unit/agents/gemini/test_gemini_bridge_cancel.py +++ b/tst/unit/agents/gemini/test_gemini_bridge_cancel.py @@ -21,7 +21,7 @@ def _state(pid: int) -> str: try: with open(f"/proc/{pid}/stat") as stat: return stat.read().rsplit(")", 1)[1].split()[0] - except FileNotFoundError: + except (FileNotFoundError, ProcessLookupError): # a process reaped mid-read fails the read with ESRCH return "" return subprocess.run(["ps", "-o", "stat=", "-p", str(pid)], capture_output=True, text=True).stdout.strip()[:1] From 93b7fea2703196b92643fc02959f957fecedefde Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 25 Oct 2026 16:03:13 -0700 Subject: [PATCH 09/10] refactor(bundle): keep write_bundle private, since only the validator writes a bundle without running it Co-Authored-By: Claude Opus 5.5 (1M context) --- AGENTS.md | 6 +++--- src/agent_env/a2a_agent/validator.py | 6 +++--- src/agent_env/bundle/__init__.py | 3 +-- src/agent_env/bundle/run.py | 18 +++++------------- tst/unit/bundle/preflight_test.py | 5 +++-- tst/unit/cli/a2a_agent_validate_test.py | 2 +- 6 files changed, 16 insertions(+), 24 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3888ad28..3cf5e06b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -165,9 +165,9 @@ Out-of-tree code plugs in either through `config.toml`, as a `module:Class` poin alias names (`Env.stored_id`). An alias names its env only where namespace routing is on. An env store refuses to create an env under an installed env alias's id (`EnvStore.put_document`), so a deployment recorded under an alias always reloads the alias's env; an env stored under such an id before keeps taking versions. - `A2AAgent.validate()` deploys `mail` (`A2AAgent.VALIDATION_ENV_ID`) with namespace routing on, writing the example - with `bundle.write_bundle` (a bundle's writes and infra, no run) first when the store has no env of that id: on a - remote sandbox its image is context-only and built in the VM, so a service like the hub's worker needs no Docker. + `A2AAgent.validate()` deploys `mail` (`A2AAgent.VALIDATION_ENV_ID`) with namespace routing on, first writing the + example (its writes and infra, no run) when the store has no env of that id: on a remote sandbox its image is + context-only and built in the VM, so a service like the hub's worker needs no Docker. - Environment providers have no table: an installed package registers one as an `agent_env.env_providers` entry point named its `type`, which records carry as `env_provider_type`. - `[plugins.]` is reserved for an installed plugin's own settings, read with diff --git a/src/agent_env/a2a_agent/validator.py b/src/agent_env/a2a_agent/validator.py index 865c0fc6..6687deba 100644 --- a/src/agent_env/a2a_agent/validator.py +++ b/src/agent_env/a2a_agent/validator.py @@ -30,7 +30,7 @@ ) from agent_env.a2a_agent.staging import transfer_store from agent_env.a2a_agent.a2a_agent import A2AAgent -from agent_env.bundle import write_bundle +from agent_env.bundle.run import _write_bundle from agent_env.config import get_config from agent_env.env.store import get_env_store from agent_env.plugins._bundles import ENV_ALIASES, installed_aliases @@ -129,7 +129,7 @@ class ModalityProbe: def write_validation_env(on_progress: Callable[[str], None]) -> None: """Write the env the validator deploys, ``A2AAgent.VALIDATION_ENV_ID``, when the store has none of that id: the env - an installed bundle gives that alias, with the infra it deploys on (``bundle.write_bundle``).""" + an installed bundle gives that alias, with the infra it deploys on.""" env_id = A2AAgent.VALIDATION_ENV_ID try: get_env_store().get(env_id) @@ -140,7 +140,7 @@ def write_validation_env(on_progress: Callable[[str], None]) -> None: if named is None: # deploying it says why it can't be found return on_progress(f"{env_id} → {named.target_id} (the alias {named.declared_by} gives it): writing it") - write_bundle(named.bundle.root, id_root=named.bundle.id_root, on_progress=on_progress) + _write_bundle(named.bundle.root, id_root=named.bundle.id_root, on_progress=on_progress) class A2AAgentValidator: diff --git a/src/agent_env/bundle/__init__.py b/src/agent_env/bundle/__init__.py index ddc1c99c..49ed558c 100644 --- a/src/agent_env/bundle/__init__.py +++ b/src/agent_env/bundle/__init__.py @@ -1,7 +1,7 @@ """Bundles: folders of entities that run without a registry.""" from .parse import Bundle, BundleEntry, BundleError, BundleKind, is_ignored, parse_bundle -from .run import BundleRun, DryRun, EvalRun, Outcome, RunInterrupted, TaskRun, dry_run_bundle, run_bundle, write_bundle +from .run import BundleRun, DryRun, EvalRun, Outcome, RunInterrupted, TaskRun, dry_run_bundle, run_bundle __all__ = [ "Bundle", @@ -18,5 +18,4 @@ "is_ignored", "parse_bundle", "run_bundle", - "write_bundle", ] diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index d3801146..9c52a8d7 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -251,22 +251,14 @@ def run_bundle( raise RunInterrupted(result, interrupts.signum) -def write_bundle( - root: Path | str, - *, - tasks: Sequence[str] = (), - evals: Sequence[str] = (), - sandbox: str | None = None, - on_progress: Callable[[str], None] | None = None, - id_root: str | None = None, +def _write_bundle( + root: Path | str, *, id_root: str | None = None, on_progress: Callable[[str], None] | None = None, ) -> Materialization: - """Write what the tasks and evals selected in ``root`` need and set up the infra envs they deploy on, as - ``run_bundle`` does before it runs them, and run nothing. Raises what ``run_bundle`` raises before a task runs.""" - if sandbox: - build_sandbox_provider(sandbox) + """Write what the bundle in ``root`` needs and set up the infra envs its tasks deploy on, as ``run_bundle`` does + before it runs them, and run nothing. Raises what ``run_bundle`` raises before a task runs.""" say = _progress(on_progress) with namespace_routing(): - plan, preflight = _planned(root, tasks, evals, id_root, sandbox, None) + plan, preflight = _planned(root, (), (), id_root, None, None) _say_agents(plan, say) return _written_with_infra(plan, preflight, say) diff --git a/tst/unit/bundle/preflight_test.py b/tst/unit/bundle/preflight_test.py index 818b3ead..274446b9 100644 --- a/tst/unit/bundle/preflight_test.py +++ b/tst/unit/bundle/preflight_test.py @@ -17,7 +17,8 @@ from agent_env.artifact.artifacts import docker_image as docker_image_module from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.store import get_artifact_store -from agent_env.bundle import BundleError, BundleKind, dry_run_bundle, run_bundle, write_bundle +from agent_env.bundle import BundleError, BundleKind, dry_run_bundle, run_bundle +from agent_env.bundle.run import _write_bundle from agent_env.cli import cli from agent_env.config import get_config from agent_env.config.runtime import Config @@ -1006,7 +1007,7 @@ async def deploy(self, context): monkeypatch.setattr(DeployEnvTaskStep, "execute", deploy) - materialization = write_bundle(bundle_dir, on_progress=lambda line: events.append(("say", line))) + materialization = _write_bundle(bundle_dir, on_progress=lambda line: events.append(("say", line))) assert events == [("say", "tasks/t.json: v1 (new)"), ("bootstrap", ["gateway", "service-db"])] assert [done.write.kind for done in materialization.writes] == [BundleKind.TASK] diff --git a/tst/unit/cli/a2a_agent_validate_test.py b/tst/unit/cli/a2a_agent_validate_test.py index a4433de1..c44d7faf 100644 --- a/tst/unit/cli/a2a_agent_validate_test.py +++ b/tst/unit/cli/a2a_agent_validate_test.py @@ -25,7 +25,7 @@ def written(monkeypatch, local_stores): """The bundles the validator writes, as (folder, id_root).""" calls = [] - monkeypatch.setattr(validator_module, "write_bundle", + monkeypatch.setattr(validator_module, "_write_bundle", lambda root, **kwargs: calls.append((root.name, kwargs["id_root"]))) return calls From 5694ca466bb8680369238e9aa936575111878d93 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 25 Oct 2026 20:16:42 -0700 Subject: [PATCH 10/10] fix(local-sandbox): teardown follows each process it signalled until it exits Teardown took a scan that no longer found a process's marker to mean it had stopped. Linux hides a process's environment while it execs and once it starts exiting, so a scan right after SIGTERM routinely misses a process that is still alive. Teardown then returned at once, never sent SIGKILL, and removed what was staged in /tmp while the process could still be running. It now keeps each process it found, by pid and start time, until /proc shows it exited. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../sandbox_providers/local_sandbox.py | 29 +++++++++++--- .../sandbox_providers/local_sandbox_test.py | 38 +++++++++++++------ 2 files changed, 51 insertions(+), 16 deletions(-) diff --git a/src/agent_env/providers/sandbox_providers/local_sandbox.py b/src/agent_env/providers/sandbox_providers/local_sandbox.py index 7bcd208d..a7995b09 100644 --- a/src/agent_env/providers/sandbox_providers/local_sandbox.py +++ b/src/agent_env/providers/sandbox_providers/local_sandbox.py @@ -511,22 +511,41 @@ def _marked_pids(sandbox_id: str) -> list[int]: async def _stop_marked(sandbox_id: str) -> bool: """Stop the processes ``sandbox_id``'s commands left running, which a real VM would take down with it: SIGTERM, then SIGKILL for any still there a few seconds later. False if some are still running.""" - if not (pids := await asyncio.to_thread(_marked_pids, sandbox_id)): + if not (found := await asyncio.to_thread(_running_marked, sandbox_id, {})): return True - logger.info("Stopping %d process(es) sandbox %s left running on this machine", len(pids), sandbox_id) + logger.info("Stopping %d process(es) sandbox %s left running on this machine", len(found), sandbox_id) for sig in (signal.SIGTERM, signal.SIGKILL): - for pid in pids: + for pid in found: with contextlib.suppress(ProcessLookupError, PermissionError): os.kill(pid, sig) for _ in range(_REAP_SECONDS * 10): await asyncio.sleep(0.1) - if not (pids := await asyncio.to_thread(_marked_pids, sandbox_id)): + if not (found := await asyncio.to_thread(_running_marked, sandbox_id, found)): return True logger.warning("Processes %s that sandbox %s started are still running; keeping what it staged in /tmp", - pids, sandbox_id) + sorted(found), sandbox_id) return False +def _running_marked(sandbox_id: str, found: dict[int, int]) -> dict[int, int]: + """The running processes carrying ``sandbox_id``'s marker, and those of ``found`` that haven't exited, each with its + start time. One found before counts until it exits: its environment reads empty while it execs or is exiting.""" + running = {pid: start for pid, start in found.items() if _started(pid) == start} + for pid in _marked_pids(sandbox_id): + if pid not in running and (start := _started(pid)) is not None: + running[pid] = start + return running + + +def _started(pid: int) -> int | None: + """When ``pid`` started, in clock ticks since boot, or None once it has exited (a zombie, or reaped).""" + try: + fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + except (FileNotFoundError, ProcessLookupError): # a process that exits mid-read gives ESRCH + return None + return None if fields[0] in "ZX" else int(fields[19]) + + def _remove_staged(sandbox_id: str) -> None: """Remove what steps staged in /tmp for ``sandbox_id``, such as a host install's work dir, which a host agent may run from until it is stopped. Steps name these with ``scoped_name``, so they end in the sandbox's id.""" diff --git a/tst/unit/providers/sandbox_providers/local_sandbox_test.py b/tst/unit/providers/sandbox_providers/local_sandbox_test.py index 89ae47e8..488cfb86 100644 --- a/tst/unit/providers/sandbox_providers/local_sandbox_test.py +++ b/tst/unit/providers/sandbox_providers/local_sandbox_test.py @@ -912,6 +912,7 @@ async def test_teardown_sends_sigterm_then_sigkill_to_what_outlives_it(monkeypat sent, running = [], {11: signal.SIGTERM, 12: signal.SIGKILL} # pid -> the signal that stops it monkeypatch.setattr(ls, "_REAP_SECONDS", 1) monkeypatch.setattr(ls, "_marked_pids", lambda sandbox_id: sorted(running) if sandbox_id == "local-a" else []) + monkeypatch.setattr(ls, "_started", lambda pid: 1 if pid in running else None) def kill(pid, sig): sent.append((pid, sig)) @@ -928,6 +929,7 @@ def kill(pid, sig): async def test_teardown_keeps_the_staging_a_surviving_process_runs_from(monkeypatch, tmp_path: Path): monkeypatch.setattr(ls, "_REAP_SECONDS", 1) monkeypatch.setattr(ls, "_marked_pids", lambda sandbox_id: [13]) # never stops + monkeypatch.setattr(ls, "_started", lambda pid: 1) monkeypatch.setattr(ls.os, "kill", lambda pid, sig: None) sandbox = _RecordingLocalSandbox(work_dir=tmp_path) staged = Path(tempfile.mkdtemp(prefix="install-agent-solver-", suffix=f"-{sandbox.sandbox_id}", dir="/tmp")) @@ -947,15 +949,6 @@ def _running(pid: int) -> bool: return False -async def _stopped(pid: int, within: float = 5.0) -> bool: - """Whether ``pid`` stops running within ``within`` seconds: a kill lands a moment after it's sent, longer on a - loaded machine.""" - async with asyncio.timeout(within): - while _running(pid): - await asyncio.sleep(0.05) - return True - - async def _detached(sandbox: LocalSandbox, command: str = "sleep 300") -> int: """Start ``command`` the way a host install starts its agent, outliving the exec that started it, and wait until it reads as marked: while a process execs, /proc shows it no environment, which on a loaded machine can last.""" @@ -997,7 +990,7 @@ async def test_teardown_stops_what_a_command_left_running_and_only_that(tmp_path await _RecordingLocalSandbox(work_dir=ours.work_dir, sandbox_id=ours.sandbox_id).terminate() - assert await _stopped(left) + assert not _running(left) assert _running(other) finally: await ls._stop_marked(theirs.sandbox_id) @@ -1012,10 +1005,33 @@ async def test_teardown_kills_a_process_that_ignores_sigterm(tmp_path: Path, mon await ls._stop_marked(sandbox.sandbox_id) - assert await _stopped(stubborn) + assert not _running(stubborn) assert not ls._marked_pids(sandbox.sandbox_id) +@linux_only +@pytest.mark.asyncio +async def test_teardown_follows_a_process_it_found_until_it_exits_though_its_environment_reads_empty(monkeypatch): + """A process's environment reads empty while it execs or is exiting, so the next scan missing it doesn't mean it + stopped.""" + monkeypatch.setattr(ls, "_REAP_SECONDS", 1) + stubborn = subprocess.Popen(["bash", "-c", "trap '' TERM; exec sleep 300"]) + try: + for _ in range(500): + if Path(f"/proc/{stubborn.pid}/cmdline").read_text().split("\0")[0] == "sleep": # exec'd, still ignoring SIGTERM + break + await asyncio.sleep(0.01) + scans = iter([[stubborn.pid]]) + monkeypatch.setattr(ls, "_marked_pids", lambda sandbox_id: next(scans, [])) + + assert await ls._stop_marked("local-a") + + assert not _running(stubborn.pid) + finally: + stubborn.kill() + stubborn.wait() + + @pytest.mark.asyncio async def test_a_container_is_created_given_the_local_ca_then_started(tmp_path, monkeypatch, copies): """The CA's trust files are in place before the container's first process runs, and its TLS clients