diff --git a/.github/workflows/sdk-smoke.yml b/.github/workflows/sdk-smoke.yml index 8589e0e0d..e0474b5ac 100644 --- a/.github/workflows/sdk-smoke.yml +++ b/.github/workflows/sdk-smoke.yml @@ -91,6 +91,7 @@ jobs: tests/providers tests/harness/test_scenario_source_provision_agent.py tests/harness/test_framework_detection.py + tests/harness/test_openai_agents_reference.py tests/harness/test_hosted_entrypoint.py tests/harness/test_hosted_entrypoint_provision_retry.py tests/harness/test_instrumentation_evidence.py diff --git a/examples/harness/openai_agents_support/.python-version b/examples/harness/openai_agents_support/.python-version new file mode 100644 index 000000000..e4fba2183 --- /dev/null +++ b/examples/harness/openai_agents_support/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/examples/harness/openai_agents_support/README.md b/examples/harness/openai_agents_support/README.md new file mode 100644 index 000000000..4bc2fc99d --- /dev/null +++ b/examples/harness/openai_agents_support/README.md @@ -0,0 +1,21 @@ +# OpenAI Agents SDK support agent + +A customer support chat agent built on the OpenAI Agents SDK. It has three tools over a +small in-memory set of orders: `look_up_order`, `start_return` and `return_policy`. + +## How the kit runs it + +The kit calls `agent_callback` in `agent.py` once per turn with the whole conversation. The +function runs the agent through `Runner.run` and returns `{"content": ...}` only. Tool calls +are not returned: the SDK runs the tools itself, and the kit's tool tracer records each call +made to the three tool functions. + +## Environment + +- `OPENAI_API_KEY`: the key for the endpoint (required for a turn). +- `OPENAI_BASE_URL`: the endpoint's base URL (required whenever a key is set; a turn is refused otherwise, so the key is never sent to the OpenAI API by default). Set it to the model gateway for the default model. +- `OPENAI_MODEL`: the model name, default `vertex_ai/gemini-3.7-flash` (a gateway name). + +The endpoint must speak chat completions. Install with `pip install -r requirements.txt` on +Python 3.12. `agent-simulate` is in the requirements because the kit's callback bridge imports +`fi.simulate.AgentInput`. diff --git a/examples/harness/openai_agents_support/agent.py b/examples/harness/openai_agents_support/agent.py new file mode 100644 index 000000000..da23af2b2 --- /dev/null +++ b/examples/harness/openai_agents_support/agent.py @@ -0,0 +1,90 @@ +import os + +from agents import ( + Agent, + OpenAIChatCompletionsModel, + RunConfig, + Runner, + function_tool, + set_tracing_disabled, +) +from openai import AsyncOpenAI + +MODEL = os.getenv("OPENAI_MODEL", "vertex_ai/gemini-3.7-flash") + +# The SDK exports its own traces to OpenAI by default, using OPENAI_API_KEY, which here is +# the gateway's key: off, so no trace or key is sent to OpenAI. +set_tracing_disabled(True) + +_ORDERS = { + "A1001": {"item": "trail running shoes", "status": "delivered", "return": None}, + "A1002": {"item": "rain jacket", "status": "shipped", "return": None}, + "A1003": {"item": "water bottle", "status": "delivered", "return": None}, +} + + +@function_tool +def look_up_order(order_id: str) -> str: + """Look up an order by its id and report its item, status and any return.""" + order = _ORDERS.get(order_id) + if order is None: + return f"No order {order_id} exists." + returned = order["return"] or "none" + return f"Order {order_id}: {order['item']}, status {order['status']}, return: {returned}." + + +@function_tool +def start_return(order_id: str, reason: str) -> str: + """Start a return for a delivered order, giving the customer's reason.""" + order = _ORDERS.get(order_id) + if order is None: + return f"No order {order_id} exists." + if order["status"] != "delivered": + return f"Order {order_id} is {order['status']} and cannot be returned yet." + if order["return"] is not None: + return f"Order {order_id} already has a return started." + order["return"] = reason + return f"Return started for order {order_id}: {reason}." + + +@function_tool +def return_policy() -> str: + """State the store's return policy.""" + return "Delivered items can be returned within 30 days for a full refund." + + +agent = Agent( + name="Support agent", + instructions=( + "You are a customer support agent for an online store. Use the tools to look up " + "orders and start returns, and never guess an order's status. Answer briefly." + ), + tools=[look_up_order, start_return, return_policy], +) + + +async def agent_callback(agent_input): + # The runner puts the new user message at the end of `messages`, so the list is the + # whole conversation as it stands. + input_list = [ + {"role": message["role"], "content": message["content"]} + for message in agent_input.messages + ] + api_key = os.getenv("OPENAI_API_KEY") + base_url = os.getenv("OPENAI_BASE_URL") + # With no base URL the client falls back to api.openai.com and would send the key there. + if api_key and not base_url: + raise RuntimeError( + "OPENAI_API_KEY is set but OPENAI_BASE_URL is empty: set OPENAI_BASE_URL to the " + "endpoint the key belongs to." + ) + # Each request runs in its own event loop, and the SDK does not close a client it was + # handed, so the client lives and dies with the call. + async with AsyncOpenAI(base_url=base_url, api_key=api_key) as client: + # A model object, because the SDK's default provider rejects a prefixed name + # such as vertex_ai/..., and this endpoint speaks chat completions. + model = OpenAIChatCompletionsModel(model=MODEL, openai_client=client) + result = await Runner.run(agent, input_list, run_config=RunConfig(model=model)) + # Content only: tool calls in the reply would be run by the caller against its own + # world and the agent called again. + return {"content": str(result.final_output)} diff --git a/examples/harness/openai_agents_support/requirements.txt b/examples/harness/openai_agents_support/requirements.txt new file mode 100644 index 000000000..c6ecb07f0 --- /dev/null +++ b/examples/harness/openai_agents_support/requirements.txt @@ -0,0 +1,2 @@ +openai-agents==0.24.0 +agent-simulate==0.1.3 diff --git a/tests/harness/reference_agents.py b/tests/harness/reference_agents.py new file mode 100644 index 000000000..c5ddb027f --- /dev/null +++ b/tests/harness/reference_agents.py @@ -0,0 +1,284 @@ +"""Shared helpers for the test files of the framework reference agents under +`examples/harness/`. This module is not a test file; it is the only one importing the private +authoring helpers of `test_bundle_author_v2`. +""" + +from __future__ import annotations + +import contextlib +import importlib +import json +import os +import shutil +import subprocess +import sys +from collections.abc import Callable, Iterator, Mapping, Sequence +from pathlib import Path + +from test_bundle_author_v2 import ( + _authoring, + _github_job, + _write_callable_contract, + _write_chat_contract, +) + +from fi.alk.harness.bundle_author_v2 import author_bundle_v2 +from fi.alk.harness.framework_detection import detect_framework, folder_findings + +REPO = Path(__file__).resolve().parents[2] +TRACE_HOOK = REPO / "src/fi/alk/harness/livekit_tool_trace_bootstrap.py" +TOOLS = ("look_up_order", "start_return", "return_policy") +# The contract writers a test file passes to `build_control_service`. +write_callable_contract = _write_callable_contract +write_chat_contract = _write_chat_contract +DETECTION_CASES = ("run_folder", "subfolder", "in_place", "shadowed") + + +# Copying and stand-ins ---------------------------------------------------------------------- + + +def copy_example(src: Path, dest: Path, ignore: Sequence[str] = ()) -> Path: + """Copy an example folder to `dest`, leaving out bytecode and any `ignore` patterns.""" + shutil.copytree(src, dest, ignore=shutil.ignore_patterns("__pycache__", *ignore)) + return dest + + +def write_stand_ins(root: Path, files: Mapping[str, str]) -> Path: + """Write `{relative_path: source}` under `root`, making the folders in between.""" + for relative, source in files.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source, encoding="utf-8") + return root + + +# Importing the example ---------------------------------------------------------------------- + + +def _top_level(modules: Sequence[str]) -> set[str]: + return {name.split(".")[0] for name in modules} + + +@contextlib.contextmanager +def fresh_import( + tmp_path: Path, + monkeypatch, + example: Path, + *, + module: str = "agent", + stand_in: Mapping[str, str] | None = None, + set_env: Mapping[str, str] | None = None, + clear_env: Sequence[str] = (), + purge: Sequence[str] = (), + copy_package: bool = False, + reset_each_load: bool = False, + ignore: Sequence[str] = (), +) -> Iterator[Callable[..., object]]: + """Yield `load(**env)`, which imports `module` of the example afresh. + + - `stand_in`: `{relative_path: source}` of the stand-in libraries, put first on the path; + none when the real library is used. + - `clear_env` is removed and `set_env` set before the first load. `load(**env)` sets `env` + first, and with `reset_each_load` clears `clear_env` before that, so each import sees + only the environment it was given. + - `purge`: top-level module names dropped from `sys.modules` before every load, with the + example's own, so one test can import twice under different environments. + - `copy_package`: copy the whole example (minus `ignore`) for every load, for a package + such as an ADK app; otherwise only `agent.py` is copied, once. + Everything imported is removed again at teardown. + """ + saved = dict(sys.modules) + clear_names = tuple(clear_env) + purged = _top_level([*purge, module]) + + def reset_environment() -> None: + for name in clear_names: + # Set, then removed, so that on teardown a variable the example sets at import is + # taken out again instead of leaking into the next test. + monkeypatch.setenv(name, "placeholder") + monkeypatch.delenv(name) + + def purge_modules() -> None: + for name in list(sys.modules): + if name.split(".")[0] in purged: + del sys.modules[name] + + reset_environment() + for name, value in (set_env or {}).items(): + monkeypatch.setenv(name, value) + run = tmp_path / "run" + if not copy_package: + run.mkdir() + shutil.copy2(example / "agent.py", run / "agent.py") + monkeypatch.syspath_prepend(str(run)) + if stand_in is not None: + monkeypatch.syspath_prepend(str(write_stand_ins(tmp_path / "stand-in", stand_in))) + purge_modules() + copies: list[Path] = [] + + def load(**environment: str): + if reset_each_load: + reset_environment() + for name, value in environment.items(): + monkeypatch.setenv(name, value) + purge_modules() + if copy_package: + copy = copy_example(example, tmp_path / f"run-{len(copies)}", ignore) + copies.append(copy) + monkeypatch.syspath_prepend(str(copy)) + return importlib.import_module(module) + + try: + yield load + finally: + for name in list(sys.modules): + if name not in saved: + del sys.modules[name] + sys.modules.update(saved) + + +# Detection ---------------------------------------------------------------------------------- + + +def assert_detected( + example: Path, + framework: str, + shadow_name: str | Sequence[str], + *, + where: str, + tmp_path: Path, + subfolder: str, + ignore: Sequence[str] = (), +) -> None: + """Check one way the example is found as `framework`; `where` is one of `DETECTION_CASES`. + + - `run_folder`: the example is the run folder, with no blocker. + - `subfolder`: it sits inside a bigger tree that also holds a package named `shadow_name` + deep down, and is found with no blocker. + - `in_place`: in the repository, with nothing named `shadow_name` on its path or inside. + - `shadowed`: a module named `shadow_name` in the folder makes the result not the + expected one, a blocker or another name, so the cases above prove something. + `shadow_name` may be several names, the first being the one planted. + """ + names = (shadow_name,) if isinstance(shadow_name, str) else tuple(shadow_name) + expected = {"name": framework, "version": None} + if where == "run_folder": + run = copy_example(example, tmp_path / "run", ignore) + assert detect_framework(run) == expected + assert folder_findings(run).blockers == frozenset() + elif where == "subfolder": + repo = tmp_path / "repo" + copy_example(example, repo / subfolder, ignore) + # A package of the same name deep in the tree must not make the library import look + # like the customer's own module. + nested = repo / "src/pkg" / names[0] + nested.mkdir(parents=True) + (nested / "__init__.py").write_text("", encoding="utf-8") + (nested / "score.py").write_text("def score():\n return 1\n", encoding="utf-8") + (repo / "main.py").write_text("print('hello')\n", encoding="utf-8") + assert detect_framework(repo / subfolder, repo_root=repo) == expected + assert folder_findings(repo / subfolder, repo_root=repo).blockers == frozenset() + elif where == "in_place": + shadows = {part for name in names for part in (name, f"{name}.py")} + assert not shadows & set(example.relative_to(REPO).parts) + assert not [ + path + for path in example.rglob("*") + if path.name in shadows and "__pycache__" not in path.parts + ] + assert detect_framework(example, repo_root=REPO) == expected + assert folder_findings(example, repo_root=REPO).blockers == frozenset() + elif where == "shadowed": + run = copy_example(example, tmp_path / "run", ignore) + assert detect_framework(run) == expected + (run / f"{names[0]}.py").write_text("def helper():\n return 1\n", encoding="utf-8") + found = detect_framework(run) + blockers = folder_findings(run).blockers + assert not (found == expected and blockers == frozenset()) + else: + raise ValueError(f"unknown case {where!r}") + + +# The build ---------------------------------------------------------------------------------- + + +def build_control_service( + tmp_path: Path, + contract_writer: Callable[[Path], None], + module: str, + *, + subfolder: str, + example: Path, + tools: Sequence[str] = TOOLS, + ignore: Sequence[str] = (), +): + """Build the example through the hosted path and return its control-service process. + + `contract_writer(authoring)` writes the contract (wrap `write_callable_contract` or + `write_chat_contract`); the tools are then declared as entrypoints of `module`. + """ + source = tmp_path / "source" + copy_example(example, source / subfolder, ignore) + (source / "README.md").write_text("many examples\n", encoding="utf-8") + authoring = _authoring(tmp_path) + contract_writer(authoring) + contract_path = authoring / "contract.json" + contract = json.loads(contract_path.read_text(encoding="utf-8")) + contract["tool_entrypoints"] = [ + {"tool": name, "mode": "import", "module": module, "callable": name} + for name in tools + ] + contract_path.write_text(json.dumps(contract), encoding="utf-8") + bundle = author_bundle_v2( + source=source, + job=_github_job( + subfolder, + with_secrets=True, + secret_aliases=("OPENAI_API_KEY", "OPENAI_BASE_URL"), + ), + authoring=authoring, + output=tmp_path / "bundle", + ) + return next(p for p in bundle.processes if p.name == bundle.runtime.control_service) + + +# The tracer --------------------------------------------------------------------------------- + + +def run_tracer( + argv: Sequence[str], + hook_dir: Path, + cwd: Path, + env: Mapping[str, str | None], + timeout: int = 180, +) -> subprocess.CompletedProcess[str]: + """Run `argv` with the hook folder first on `PYTHONPATH`, and fail with its stderr. + + `env` is added to the current environment; a value of None removes the variable. Any + `PYTHONPATH` in it follows the hook folder. + """ + environment = {**os.environ} + for name, value in env.items(): + if value is None: + environment.pop(name, None) + else: + environment[name] = value + paths = [str(hook_dir), *filter(None, environment.get("PYTHONPATH", "").split(os.pathsep))] + environment["PYTHONPATH"] = os.pathsep.join(paths) + completed = subprocess.run( + list(argv), + capture_output=True, + text=True, + cwd=cwd, + env=environment, + timeout=timeout, + ) + assert completed.returncode == 0, completed.stderr[-3000:] + return completed + + +def assert_pins(folder: Path, expected_lines: Sequence[str], python: str = "3.12") -> None: + """The example pins exactly `expected_lines` in requirements.txt and `python`.""" + lines = (folder / "requirements.txt").read_text(encoding="utf-8").splitlines() + assert lines == list(expected_lines) + assert (folder / ".python-version").read_text(encoding="utf-8").strip() == python diff --git a/tests/harness/test_openai_agents_reference.py b/tests/harness/test_openai_agents_reference.py new file mode 100644 index 000000000..f525fb336 --- /dev/null +++ b/tests/harness/test_openai_agents_reference.py @@ -0,0 +1,394 @@ +"""The OpenAI Agents SDK reference agent, and how the kit's hosted path handles it. + +The SDK is never installed here. Where the example has to be imported, a stand-in `agents` +and `openai` written below take its place, shaped like the public names the example uses. +""" + +from __future__ import annotations + +import ast +import asyncio +import json +import os +import shutil +import sys +import textwrap +from pathlib import Path + +import pytest +from reference_agents import ( + DETECTION_CASES, + REPO, + TOOLS, + TRACE_HOOK, + assert_detected, + assert_pins, + build_control_service, + fresh_import, + run_tracer, + write_callable_contract, + write_stand_ins, +) + +from fi.simulate import AgentInput + +SUBFOLDER = "examples/harness/openai_agents_support" +EXAMPLE = REPO / SUBFOLDER +ENVIRONMENT = ("OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENAI_MODEL") + +# `function_tool` and `on_invoke_tool` follow SDK 0.24.0 (tool.py:2761, 2823-2832): a plain +# function is run with `asyncio.to_thread`, a coroutine function is awaited. +STAND_IN_AGENTS = ''' +import asyncio +import inspect +import json +from types import SimpleNamespace + +STATE = { + "tracing_disabled": [], + "agents": [], + "runs": [], + "final_output": "All done.", + "raise": None, +} + + +def set_tracing_disabled(disabled): + STATE["tracing_disabled"].append(disabled) + + +class FunctionTool: + def __init__(self, func): + self.name = func.__name__ + self.func = func + + async def on_invoke_tool(self, ctx, input_json): + args = json.loads(input_json) if input_json else {} + if inspect.iscoroutinefunction(self.func): + return await self.func(**args) + return await asyncio.to_thread(self.func, **args) + + +def function_tool(func=None, **options): + if func is None: + return FunctionTool + return FunctionTool(func) + + +class Agent: + def __init__(self, name, instructions=None, tools=(), **options): + self.name = name + self.tools = list(tools) + self.options = options + STATE["agents"].append(self) + + +class OpenAIChatCompletionsModel: + def __init__(self, model, openai_client): + self.model = model + self.openai_client = openai_client + + +class RunConfig: + def __init__(self, model=None, **options): + self.model = model + + +class Runner: + @classmethod + async def run(cls, starting_agent, input, *, run_config=None, **options): + client = getattr(getattr(run_config, "model", None), "openai_client", None) + STATE["runs"].append( + SimpleNamespace( + agent=starting_agent, + input=input, + run_config=run_config, + client_open=client is not None and not client.closed, + ) + ) + if STATE["raise"] is not None: + raise STATE["raise"] + return SimpleNamespace(final_output=STATE["final_output"]) +''' + +# The real client refuses to be built without a key, which is what makes a client built at +# import fail an import in an environment that has none. +STAND_IN_OPENAI = ''' +CLIENTS = [] + + +class AsyncOpenAI: + def __init__(self, base_url=None, api_key=None): + if api_key is None: + raise RuntimeError("The api_key client option must be set") + self.base_url = base_url + self.api_key = api_key + self.closed = False + CLIENTS.append(self) + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_info): + self.closed = True +''' + + +STAND_IN_FILES = {"agents/__init__.py": STAND_IN_AGENTS, "openai/__init__.py": STAND_IN_OPENAI} + + +@pytest.fixture +def example_module(tmp_path, monkeypatch): + """A fresh import of the example's `agent.py` on the stand-in SDK, with no key set.""" + with fresh_import( + tmp_path, + monkeypatch, + EXAMPLE, + stand_in=STAND_IN_FILES, + clear_env=ENVIRONMENT, + purge=("agents", "openai"), + ) as load: + yield load + + +# T1: detection ------------------------------------------------------------------------------ + + +@pytest.mark.parametrize("where", DETECTION_CASES) +def test_example_is_detected_as_the_sdk_agent(tmp_path: Path, where: str) -> None: + # `agents` is the SDK's top-level package, so a package or module of that name anywhere + # near the example would pass for the customer's own. + assert_detected( + EXAMPLE, "openai_agents", "agents", where=where, tmp_path=tmp_path, subfolder=SUBFOLDER + ) + + +# T2: the build ------------------------------------------------------------------------------ + + +def test_build_runs_the_callback_bridge_and_binds_the_tool_tracer(tmp_path: Path) -> None: + agent = build_control_service( + tmp_path, write_callable_contract, "agent", subfolder=SUBFOLDER, example=EXAMPLE + ) + + assert "ThreadingHTTPServer" in json.dumps(agent.run_command) + assert agent.environment["ALK_CALLBACK_ENTRYPOINT"] == "agent:agent_callback" + assert agent.environment["PORT"] == "{{PORT_agent}}" + assert "requirements.txt" in json.dumps(agent.build_commands) + assert agent.environment["HARNESS_TOOL_TRACE"] == "{{WORLD_DIR}}/agent-tool-calls.jsonl" + bindings = json.loads(agent.environment["ALK_TOOL_TRACE_BINDINGS"]) + assert sorted(bindings, key=lambda item: item["name"]) == sorted( + ({"name": name, "module": "agent", "callable": name} for name in TOOLS), + key=lambda item: item["name"], + ) + + +# T3: the tracer and an SDK-shaped call ------------------------------------------------------ + +DRIVER = textwrap.dedent( + """ + import asyncio + import json + import threading + + import agent + import agent_extra + from agents import function_tool + + # Same name as a declared tool, other module: only the module tells them apart. + unlisted = function_tool(agent_extra.look_up_order) + CALLS = [ + (agent.look_up_order, {"order_id": "A1001"}), + (agent.start_return, {"order_id": "A1001", "reason": "too small"}), + (agent.return_policy, {}), + (unlisted, {"order_id": "X9"}), + ] + + + async def go(): + return [await tool.on_invoke_tool(None, json.dumps(args)) for tool, args in CALLS] + + + # The bridge serves each request on a thread started after start-up. + result = {} + worker = threading.Thread(target=lambda: result.update(out=asyncio.run(go()))) + worker.start() + worker.join() + print(json.dumps(result["out"])) + """ +) + + +def test_tools_called_the_way_the_sdk_calls_them_are_traced(tmp_path: Path) -> None: + run = tmp_path / "run" + run.mkdir() + shutil.copy2(EXAMPLE / "agent.py", run / "agent.py") + (run / "agent_extra.py").write_text( + "def look_up_order(order_id):\n return 'not declared'\n", encoding="utf-8" + ) + hook = tmp_path / "hook" + hook.mkdir() + shutil.copy2(TRACE_HOOK, hook / "sitecustomize.py") + stand_in = write_stand_ins(tmp_path / "stand-in", STAND_IN_FILES) + trace = tmp_path / "calls.jsonl" + bindings = [{"name": name, "module": "agent", "callable": name} for name in TOOLS] + + completed = run_tracer( + [sys.executable, "-c", DRIVER], + hook, + run, + { + "PYTHONPATH": os.pathsep.join([str(stand_in), str(run)]), + "HARNESS_TOOL_TRACE": str(trace), + "ALK_TOOL_TRACE_BINDINGS": json.dumps(bindings), + }, + ) + + outputs = json.loads(completed.stdout.splitlines()[-1]) + assert outputs[3] == "not declared" + records = [json.loads(line) for line in trace.read_text(encoding="utf-8").splitlines()] + assert [(record["name"], record["arguments"]) for record in records] == [ + ("look_up_order", {"order_id": "A1001"}), + ("start_return", {"order_id": "A1001", "reason": "too small"}), + ("return_policy", {}), + ] + assert [record["output"] for record in records] == outputs[:3] + assert not any(record["is_error"] for record in records) + + +# T4: the callback's glue -------------------------------------------------------------------- + +NEW_MESSAGE = {"role": "user", "content": "Please start a return for A1001."} +CONVERSATION = [ + {"role": "user", "content": "Where is my order A1001?"}, + {"role": "assistant", "content": "It was delivered on Monday."}, + NEW_MESSAGE, +] + + +def _agent_input(messages: list[dict]) -> AgentInput: + # Built as the hosted chat target builds it for each turn: the whole conversation, ending + # with the new message, and `new_message` a copy of that last item. + return AgentInput( + thread_id="scenario-one", + execution_id="run-one", + turn_index=1, + scenario_name="scenario-one", + modality="text", + messages=list(messages), + new_message=dict(messages[-1]), + tools=[], + ) + + +def test_callback_sends_the_conversation_once_and_replies_with_content_only( + example_module, monkeypatch +) -> None: + monkeypatch.setenv("OPENAI_BASE_URL", "http://gateway.test/v1") + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + agent = example_module() + import agents + import openai + + agents.STATE["final_output"] = "Return started." + # An extra key on an item is the glue's to drop; the SDK takes role and content. + messages = [{**CONVERSATION[0], "extra": "dropped"}, *CONVERSATION[1:]] + + reply = asyncio.run(agent.agent_callback(_agent_input(messages))) + + assert reply == {"content": "Return started."} + (run,) = agents.STATE["runs"] + assert run.input == CONVERSATION + assert run.input.count(NEW_MESSAGE) == 1 + assert run.agent is agent.agent + assert [tool.name for tool in agent.agent.tools] == list(TOOLS) + model = run.run_config.model + assert isinstance(model, agents.OpenAIChatCompletionsModel) + assert model.model == agent.MODEL == "vertex_ai/gemini-3.7-flash" + assert model.openai_client.base_url == "http://gateway.test/v1" + assert model.openai_client.api_key == "test-key" + assert run.client_open + assert model.openai_client.closed + assert openai.CLIENTS == [model.openai_client] + + +def test_each_call_builds_and_closes_its_own_client(example_module, monkeypatch) -> None: + monkeypatch.setenv("OPENAI_BASE_URL", "http://gateway.test/v1") + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + agent = example_module() + import agents + import openai + + asyncio.run(agent.agent_callback(_agent_input(CONVERSATION[:1]))) + asyncio.run(agent.agent_callback(_agent_input(CONVERSATION))) + + first, second = openai.CLIENTS + assert first is not second + assert first.closed and second.closed + assert [run.client_open for run in agents.STATE["runs"]] == [True, True] + assert [len(run.input) for run in agents.STATE["runs"]] == [1, 3] + assert len(agents.STATE["agents"]) == 1 + + +def test_import_sets_up_the_agent_without_a_key_or_a_client(example_module) -> None: + agent = example_module() + import agents + import openai + + assert agents.STATE["tracing_disabled"] == [True] + assert openai.CLIENTS == [] + assert agents.STATE["runs"] == [] + (built,) = agents.STATE["agents"] + assert built is agent.agent + # The model comes from the run, not the agent. + assert "model" not in built.options + + +@pytest.mark.parametrize("base_url", [None, ""]) +def test_a_key_with_no_base_url_is_refused_before_a_client_is_built( + example_module, monkeypatch, base_url +) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "secret-canary-key") + if base_url is not None: + monkeypatch.setenv("OPENAI_BASE_URL", base_url) + agent = example_module() + import agents + import openai + + with pytest.raises(RuntimeError, match="OPENAI_BASE_URL") as refused: + asyncio.run(agent.agent_callback(_agent_input(CONVERSATION))) + + assert "secret-canary-key" not in str(refused.value) + assert openai.CLIENTS == [] + assert agents.STATE["runs"] == [] + + +def test_sdk_failure_is_not_hidden(example_module, monkeypatch) -> None: + monkeypatch.setenv("OPENAI_BASE_URL", "http://gateway.test/v1") + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + agent = example_module() + import agents + import openai + + agents.STATE["raise"] = RuntimeError("model unreachable") + + with pytest.raises(RuntimeError, match="model unreachable"): + asyncio.run(agent.agent_callback(_agent_input(CONVERSATION))) + assert openai.CLIENTS[0].closed + + +def test_example_has_one_async_callback_and_plain_tools() -> None: + module = ast.parse((EXAMPLE / "agent.py").read_text(encoding="utf-8")) + async_names = [ + node.name for node in module.body if isinstance(node, ast.AsyncFunctionDef) + ] + assert async_names == ["agent_callback"] + plain = {node.name for node in module.body if isinstance(node, ast.FunctionDef)} + assert plain == set(TOOLS) + + +# T5: requirements --------------------------------------------------------------------------- + + +def test_requirements_and_python_version_are_pinned() -> None: + assert_pins(EXAMPLE, ["openai-agents==0.24.0", "agent-simulate==0.1.3"])