diff --git a/.github/workflows/sdk-smoke.yml b/.github/workflows/sdk-smoke.yml index c5ff1f7c7..cf6cb8a2b 100644 --- a/.github/workflows/sdk-smoke.yml +++ b/.github/workflows/sdk-smoke.yml @@ -72,6 +72,8 @@ jobs: tests/harness/test_process_preflight.py tests/providers tests/harness/test_scenario_source_provision_agent.py + tests/harness/test_framework_detection.py + tests/harness/test_hosted_entrypoint.py tests/harness/test_hosted_entrypoint_provision_retry.py tests/harness/test_instrumentation_evidence.py tests/harness/test_agent_type_refusal.py diff --git a/src/fi/alk/harness/framework_detection.py b/src/fi/alk/harness/framework_detection.py new file mode 100644 index 000000000..bec4b49fa --- /dev/null +++ b/src/fi/alk/harness/framework_detection.py @@ -0,0 +1,1012 @@ +"""Names the framework an agent's source is built on, from its files alone. + +The answer is a label: a wrong name costs little, a lost document or a stuck run +costs a lot. So nothing here runs or imports customer code, and any failure ends +in `unknown`. Files are read with `tokenize`, because it stays bounded on files a +parser would choke on and still reads what follows a syntax error. +""" + +from __future__ import annotations + +import glob +import io +import json +import logging +import os +import re +import stat +import tokenize +import warnings +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterator, Literal + +from .instrumentation_evidence import _SKIPPED_DIRECTORIES +from .source_folder import agent_folder + +logger = logging.getLogger(__name__) + +_MAX_LINE_BYTES = 65_536 +_MAX_TOTAL_BYTES = 67_108_864 +_MAX_FILES = 5_000 +_METADATA_BYTES = 65_536 +_VERSION_MAX = 64 +_MAX_WORK = 100_000_000_000 +_WORK_PER_TARGET = 2_000 +_MAX_ALIASES = 10_000 +# A backslash-continued dotted name has no length limit of its own, and no real +# name comes near this. +_MAX_DOTTED_PARTS = 64 + +_FILE_SKIPPED = "file_skipped" +_FILE_UNREADABLE = "file_unreadable" +_AGENTS_AMBIGUOUS = "agents_ambiguous" +_SIDE_FOLDER_FRAMEWORK = "side_folder_framework" +_OUTSIDE_IMPORT = "outside_import" +_OTHER_SOURCE = "other_source" +_NO_PYTHON = "no_python" + +# These three are common names for a customer's own package below the top. +_TOP_ONLY_SKIPS = frozenset({"build", "dist", "env"}) +_ALWAYS_SKIPPED = frozenset(_SKIPPED_DIRECTORIES) - _TOP_ONLY_SKIPS +_SIDE_FOLDERS = frozenset( + { + "tests", + "test", + "examples", + "example", + "samples", + "sample", + "demos", + "demo", + "docs", + "doc", + "evals", + "eval", + "scripts", + "legacy", + "notebooks", + "vendor", + "vendored", + "third_party", + "external", + "contrib", + "experiments", + "benchmarks", + "archive", + } +) +_OTHER_SOURCE_SUFFIXES = (".ts", ".tsx", ".ipynb") +_OTHER_SOURCE_NAME = "root_agent.yaml" + +_VERSION_PATTERN = re.compile( + r"\d+(\.\d+)*((a|b|rc)\d+)?(\.post\d+)?(\.dev\d+)?(\+[A-Za-z0-9.]+)?", + re.ASCII | re.IGNORECASE, +) + +# 3.11 has neither constant: an f-string is one STRING token there. +_FSTRING_START = getattr(tokenize, "FSTRING_START", None) +_FSTRING_END = getattr(tokenize, "FSTRING_END", None) +_HIDDEN_TOKENS = frozenset({tokenize.COMMENT, tokenize.NL, tokenize.ENCODING}) +_OPENING = frozenset("([{") +_CLOSING = frozenset(")]}") + + +@dataclass(frozen=True) +class Finding: + framework: str + strength: Literal["builds", "model", "import"] + file: str + line: int + + +@dataclass(frozen=True) +class FolderFindings: + findings: tuple[Finding, ...] + blockers: frozenset[str] + over_limit: bool + + +@dataclass(frozen=True) +class FrameworkFolders: + folder: Path + repo_root: Path | None = None + package_dir: Path | None = None + + +@dataclass(frozen=True) +class _Row: + result: str + imports: tuple[str, ...] + builders: frozenset[str] + packages: tuple[str, ...] + label: str | None = None + + @property + def framework(self) -> str: + return self.label or self.result + + +def _named( + result: str, + imports: tuple[str, ...], + builders: tuple[str, ...], + packages: tuple[str, ...], +) -> _Row: + return _Row(result, imports, frozenset(builders), packages) + + +def _other(label: str, imports: tuple[str, ...], packages: tuple[str, ...]) -> _Row: + return _Row("other", imports, frozenset(), packages, label) + + +_ROWS = ( + _named( + "deepagents", + ("deepagents",), + ("create_deep_agent", "middleware"), + ("deepagents",), + ), + _named( + "langgraph", + ("langgraph", "langgraph_*"), + ( + "StateGraph", + "MessageGraph", + "create_react_agent", + "entrypoint", + "create_supervisor", + "create_swarm", + ), + ("langgraph",), + ), + _named( + "langchain", + ("langchain", "langchain_*"), + ( + "create_agent", + "create_react_agent", + "create_tool_calling_agent", + "create_openai_tools_agent", + "create_openai_functions_agent", + "create_structured_chat_agent", + "AgentExecutor", + "initialize_agent", + ), + ("langchain", "langchain-core"), + ), + _named( + "livekit", + ("livekit.agents",), + ("AgentSession", "Agent", "WorkerOptions", "cli"), + ("livekit-agents",), + ), + _named("pipecat", ("pipecat",), ("Pipeline", "PipelineTask"), ("pipecat-ai",)), + _named( + "claude_agent_sdk", + ("claude_agent_sdk", "claude_code_sdk"), + ("query", "ClaudeSDKClient"), + ("claude-agent-sdk", "claude-code-sdk"), + ), + _named( + "openai_agents", + ("agents",), + ("Agent", "Runner", "RealtimeAgent", "RealtimeRunner"), + ("openai-agents",), + ), + _named( + "pydantic_ai", ("pydantic_ai",), ("Agent",), ("pydantic-ai", "pydantic-ai-slim") + ), + _named( + "google_adk", + ("google.adk",), + ( + "Agent", + "LlmAgent", + "SequentialAgent", + "ParallelAgent", + "LoopAgent", + "BaseAgent", + "Runner", + "InMemoryRunner", + ), + ("google-adk",), + ), + _named("crewai", ("crewai",), ("Crew", "Agent", "CrewBase", "Flow"), ("crewai",)), + _other("llama_index", ("llama_index",), ("llama-index-core", "llama-index")), + _other( + "autogen", + ("autogen", "autogen_agentchat", "autogen_core", "autogen_ext"), + ("autogen-agentchat", "pyautogen", "autogen"), + ), + _other("semantic_kernel", ("semantic_kernel",), ("semantic-kernel",)), + _other("haystack", ("haystack",), ("haystack-ai",)), + _other("agno", ("agno",), ("agno",)), + _other("beeai_framework", ("beeai_framework",), ("beeai-framework",)), + _other("dspy", ("dspy",), ("dspy", "dspy-ai")), + _other("smolagents", ("smolagents",), ("smolagents",)), + _other("strands", ("strands",), ("strands-agents",)), +) +_NAMED_FRAMEWORKS = frozenset(row.result for row in _ROWS if row.label is None) +_ROW_BY_FRAMEWORK = {row.framework: row for row in _ROWS} + + +def _index_rows() -> tuple[ + dict[str, list[tuple[_Row, tuple[str, ...]]]], tuple[tuple[str, _Row], ...] +]: + by_top: dict[str, list[tuple[_Row, tuple[str, ...]]]] = {} + wildcards: list[tuple[str, _Row]] = [] + for row in _ROWS: + for prefix in row.imports: + if prefix.endswith("*"): + wildcards.append((prefix[:-1], row)) + else: + parts = tuple(prefix.split(".")) + by_top.setdefault(parts[0], []).append((row, parts)) + return by_top, tuple(wildcards) + + +_PREFIXES_BY_TOP, _WILDCARD_TOPS = _index_rows() + + +def _row_match(parts: list[str]) -> tuple[_Row, int] | None: + """The row a dotted name falls under, and how many of its parts that prefix took.""" + top = parts[0] + for row, prefix in _PREFIXES_BY_TOP.get(top, ()): + if tuple(parts[: len(prefix)]) == prefix: + return row, len(prefix) + for start, row in _WILDCARD_TOPS: + if top.startswith(start): + return row, 1 + return None + + +def _is_row_top(top: str) -> bool: + return top in _PREFIXES_BY_TOP or any( + top.startswith(start) for start, _ in _WILDCARD_TOPS + ) + + +def _is_chat_model(name: str) -> bool: + if name == "init_chat_model": + return True + return name.startswith(("Chat", "AzureChat")) and not name.startswith( + ("ChatMessage", "ChatPrompt", "ChatGeneration", "ChatResult") + ) + + +class _OverBudget(Exception): + pass + + +class _Work: + """What one walk has spent. A count and not a clock, so the answer is the same + every time. Python 3.12.3 re-decodes a token's whole line for each token, and one + name can have thousands of targets: neither shows in the size of a file.""" + + def __init__(self) -> None: + self._spent = 0 + + def charge(self, amount: int) -> None: + self._spent += amount + if self._spent > _MAX_WORK: + raise _OverBudget() + + +def _visible_tokens(data: bytes, work: _Work) -> Iterator[tokenize.TokenInfo]: + fstring_depth = 0 + for token in tokenize.tokenize(io.BytesIO(data).readline): + work.charge(len(token.line)) + if token.type == _FSTRING_START: + fstring_depth += 1 + elif token.type == _FSTRING_END: + fstring_depth -= 1 + elif not fstring_depth and token.type not in _HIDDEN_TOKENS: + yield token + + +def _is_type_checking_header(words: list[str]) -> bool: + return words == ["if", "TYPE_CHECKING", ":"] or ( + len(words) == 5 + and words[0] == "if" + and words[2:] == [".", "TYPE_CHECKING", ":"] + ) + + +def _outside_type_checking( + tokens: Iterator[tokenize.TokenInfo], +) -> Iterator[tokenize.TokenInfo]: + """Drops an indented block under `if TYPE_CHECKING:`, because it never runs. + + Its `else` runs, and arrives here as an ordinary statement after the block's + DEDENT. A one-line `if TYPE_CHECKING: import x` is left in: that costs an + `unknown`, never a wrong name. + """ + skipping = 0 + line: list[str] = [] + header: list[str] = [] + for token in tokens: + if skipping: + if token.type == tokenize.INDENT: + skipping += 1 + elif token.type == tokenize.DEDENT: + skipping -= 1 + continue + if token.type == tokenize.INDENT and _is_type_checking_header(header): + skipping = 1 + continue + if token.type == tokenize.NEWLINE: + header, line = line, [] + elif token.type not in (tokenize.INDENT, tokenize.DEDENT) and len(line) < 6: + line.append(token.string) + yield token + + +def _ends_statement(token: tokenize.TokenInfo | None) -> bool: + return ( + token is None + or token.type in (tokenize.NEWLINE, tokenize.ENDMARKER) + or token.string == ";" + ) + + +def _skip_statement( + token: tokenize.TokenInfo | None, tokens: Iterator[tokenize.TokenInfo] +) -> None: + while not _ends_statement(token): + token = next(tokens, None) + + +def _dotted( + token: tokenize.TokenInfo | None, tokens: Iterator[tokenize.TokenInfo] +) -> tuple[list[str], tokenize.TokenInfo | None]: + """Reads `n(.n)*` from `token` on; returns its parts and the first token after it.""" + parts: list[str] = [] + while token is not None and token.type == tokenize.NAME: + if len(parts) < _MAX_DOTTED_PARTS: + parts.append(token.string) + token = next(tokens, None) + if token is None or token.string != ".": + break + token = next(tokens, None) + return parts, token + + +def _after_subscript( + tokens: Iterator[tokenize.TokenInfo], +) -> tokenize.TokenInfo | None: + """Called just past a `[`; returns the first token after its closing `]`.""" + depth = 1 + token = next(tokens, None) + while token is not None and depth: + if token.string == "[": + depth += 1 + elif token.string == "]": + depth -= 1 + token = next(tokens, None) + return token + + +class _FileScan: + """One file's two reads. Uses are resolved as they are met and never kept, so + what this holds grows with the file's imports, not with its length.""" + + def __init__( + self, shadowed: frozenset[str], outside_names: frozenset[str], work: _Work + ) -> None: + self.work = work + self._shadowed = shadowed + self._outside_names = outside_names + # Targets stay as parts: a name can be megabytes long, and a joined copy per + # target would hold far more than the source does. + self.aliases: dict[str, set[tuple[str, ...]]] = {} + self._alias_count = 0 + self.found: dict[tuple[str, str], int] = {} + self.has_row_import = False + self.imports_outside = False + + def read_imports(self, tokens: Iterator[tokenize.TokenInfo]) -> None: + # Both are keywords: `import` only ever starts an import, and `_from_import` + # skips a `yield from` or `raise ... from`, since no `import` follows them. + for token in tokens: + if token.type == tokenize.NAME and token.string == "import": + self._plain_import(token.start[0], tokens) + elif token.type == tokenize.NAME and token.string == "from": + self._from_import(token.start[0], tokens) + + def _plain_import(self, line: int, tokens: Iterator[tokenize.TokenInfo]) -> None: + while True: + parts, token = _dotted(next(tokens, None), tokens) + if parts: + if token is not None and token.string == "as": + named = next(tokens, None) + token = next(tokens, None) + if named is not None and named.type == tokenize.NAME: + self._note(parts, named.string, tuple(parts), line) + else: + self._note(parts, parts[0], (parts[0],), line) + if token is None or token.string != ",": + _skip_statement(token, tokens) + return + + def _from_import(self, line: int, tokens: Iterator[tokenize.TokenInfo]) -> None: + # A relative import starts with a `.` or `...` token, which `_dotted` stops at + # with no parts. + module, token = _dotted(next(tokens, None), tokens) + if token is None or token.string != "import": + _skip_statement(token, tokens) + return + token = next(tokens, None) + while token is not None and not _ends_statement(token): + if token.string == "*": + self._note(module, None, None, line) + elif token.type == tokenize.NAME: + imported = [*module, token.string] + alias = token.string + token = next(tokens, None) + if token is not None and token.string == "as": + named = next(tokens, None) + token = next(tokens, None) + if named is None or named.type != tokenize.NAME: + continue + alias = named.string + self._note(imported, alias, tuple(imported), line) + continue + token = next(tokens, None) + + def _note( + self, + imported: list[str], + alias: str | None, + target: tuple[str, ...] | None, + line: int, + ) -> None: + top = imported[0] + if top in self._outside_names: + self.imports_outside = True + if top in self._shadowed: + return + match = _row_match(imported) + if match is not None: + self.has_row_import = True + self.found.setdefault((match[0].framework, "import"), line) + if alias is None or target is None or not _is_row_top(top): + return + targets = self.aliases.setdefault(alias, set()) + if target not in targets: + targets.add(target) + self._alias_count += 1 + if self._alias_count > _MAX_ALIASES: + raise ValueError("too many aliases") + + def read_uses(self, tokens: Iterator[tokenize.TokenInfo]) -> None: + depth = 0 + class_depth: int | None = None + class_header = False + at_start = True + decorates = False + prev = "" + token = next(tokens, None) + while token is not None: + text = token.string + if ( + token.type == tokenize.NAME + and text in self.aliases + and prev not in (".", "def", "class") + ): + line = token.start[0] + decorator = prev == "@" and decorates + base_position = class_depth == depth and prev in ("(", ",") + parts, token = _dotted(token, tokens) + if token is not None and token.string == "[": + token = _after_subscript(tokens) + following = token.string if token is not None else "" + if decorator: + self._resolve(parts, "decorator", line) + elif following == "(": + self._resolve(parts, "call", line) + elif base_position and following in (",", ")"): + self._resolve(parts, "base", line) + else: + self._resolve(parts, "mention", line) + prev = parts[-1] + at_start = False + continue + if token.type == tokenize.NEWLINE: + depth = 0 + class_depth = None + class_header = False + at_start = True + prev = "" + elif token.type not in (tokenize.INDENT, tokenize.DEDENT): + if text in _OPENING: + depth += 1 + # Depth 1, because `class F[T: (A, B)](Base):` has brackets ahead. + if text == "(" and class_header and depth == 1: + class_depth = depth + class_header = False + elif text in _CLOSING: + depth = max(depth - 1, 0) + if class_depth is not None and depth < class_depth: + class_depth = None + elif text == "class" and token.type == tokenize.NAME: + class_header = True + elif text == ":" and depth == 0: + class_header = False + if text == "@": + decorates = at_start + at_start = text == ";" + prev = text + token = next(tokens, None) + + def _resolve(self, parts: list[str], kind: str, line: int) -> None: + targets = self.aliases[parts[0]] + self.work.charge(_WORK_PER_TARGET * len(targets)) + for target in targets: + full = [*target, *parts[1:]] + match = _row_match(full) + if match is None: + continue + row, taken = match + # `import livekit` isn't under the row, so `livekit.agents.x` is the first + # sign of it: without this the folder would say `none`. + self.found.setdefault((row.framework, "import"), line) + if kind == "mention": + continue + if row.label is not None: + # `other` means "called something from it": a base class or a + # decorator alone says too little about what the agent runs on. + if kind == "call": + self.found.setdefault((row.framework, "builds"), line) + elif any(part in row.builders for part in full[taken:]): + self.found.setdefault((row.framework, "builds"), line) + if ( + row.result == "langchain" + and full[0] != "langchain_core" + and _is_chat_model(full[-1]) + ): + self.found.setdefault((row.framework, "model"), line) + + +def _scan_file(data: bytes, scan: _FileScan, *, read_uses: bool) -> bool: + """Runs the reads; True when one failed. What was found before a failure stands.""" + failed = False + with warnings.catch_warnings(): + # On 3.12 `tokenize` warns once per line on a `\{`, which floods stderr. + warnings.simplefilter("ignore") + try: + scan.read_imports(_outside_type_checking(_visible_tokens(data, scan.work))) + except _OverBudget: + raise + except Exception: + failed = True + if read_uses and scan.aliases: + try: + scan.read_uses(_outside_type_checking(_visible_tokens(data, scan.work))) + except _OverBudget: + raise + except Exception: + failed = True + return failed + + +def _read_file(path: str, limit: int) -> bytes: + # O_NONBLOCK so that opening a named pipe can't hang the build. + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): + raise OSError("not a regular file") + chunks: list[bytes] = [] + remaining = limit + while remaining > 0: + chunk = os.read(descriptor, min(remaining, 1_048_576)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + finally: + os.close(descriptor) + + +def _has_long_line(data: bytes) -> bool: + # A very long line is what makes `tokenize` slow. `find`, because `split` + # costs hundreds of MiB on a file at the limit. + start = 0 + while True: + newline = data.find(b"\n", start) + end = len(data) if newline < 0 else newline + if end - start > _MAX_LINE_BYTES: + return True + if newline < 0: + return False + start = newline + 1 + + +def _module_names(directory: Path) -> set[str]: + names: set[str] = set() + with os.scandir(directory) as entries: + for entry in entries: + if entry.is_dir(): + names.add(entry.name) + elif entry.name.endswith(".py"): + names.add(entry.name[: -len(".py")]) + return names + + +def _outside_names( + folder: Path, repo_root: Path | None, blockers: set[str] +) -> frozenset[str]: + """Names that, imported from the folder, are shared code sitting outside it.""" + if repo_root is None: + return frozenset() + try: + here = folder.resolve() + top = repo_root.resolve() + try: + on_path = set(here.relative_to(top).parts) + except ValueError: + # The usual packaged build: the folder is a copy outside the repo. + on_path = set() + return frozenset(_module_names(top) - _module_names(here) - on_path) + except (OSError, RuntimeError): + blockers.add(_FILE_SKIPPED) + return frozenset() + + +def _own_modules(folder: Path, blockers: set[str]) -> frozenset[str]: + """Top names that are the customer's, because they sit ahead of the installed one.""" + own: set[str] = set() + installed_here: set[str] = set() + try: + with os.scandir(folder) as entries: + for entry in entries: + name = entry.name + if name.endswith(".py"): + own.add(name[: -len(".py")]) + elif name.endswith(".dist-info") and "-" in name and entry.is_dir(): + # `pip install -t .`: the package beside it is the real one. + blockers.add(_FILE_SKIPPED) + installed_here.add( + name[: -len(".dist-info")].rpartition("-")[0].replace("-", "_") + ) + elif entry.is_dir() and os.path.isfile( + os.path.join(entry.path, "__init__.py") + ): + own.add(name) + except OSError: + blockers.add(_FILE_SKIPPED) + # `agents` is too common a name for its presence to decide; see `_agents_rule`. + return frozenset(own - installed_here - {"agents"}) + + +def _is_side_part(part: str) -> bool: + return part in _SIDE_FOLDERS or part.startswith(".") + + +def _is_side_file(name: str) -> bool: + return ( + name == "conftest.py" + or (name.startswith("test_") and name.endswith(".py")) + or name.endswith("_test.py") + or name.startswith(".") + ) + + +def _directories_to_enter( + directory: str, names: list[str], *, at_top: bool, blockers: set[str] +) -> list[str]: + kept = [] + for name in names: + if name in _ALWAYS_SKIPPED or (at_top and name in _TOP_ONLY_SKIPS): + continue + if os.path.islink(os.path.join(directory, name)): + blockers.add(_FILE_SKIPPED) + continue + kept.append(name) + return kept + + +class _FolderWalk: + def __init__( + self, + shadowed: frozenset[str], + outside_names: frozenset[str], + blockers: set[str], + ) -> None: + self._shadowed = shadowed + self._outside_names = outside_names + self.blockers = blockers + self.found: dict[tuple[str, str, str], int] = {} + self.python_seen = 0 + self.bytes_read = 0 + self.has_counted_python = False + self.agents_in_counted_path = False + self.over_limit = False + self._work = _Work() + + def take(self, directory: str, relative: tuple[str, ...], name: str) -> None: + counted = not ( + any(_is_side_part(part) for part in relative) or _is_side_file(name) + ) + if not name.endswith(".py"): + if counted and ( + name.endswith(_OTHER_SOURCE_SUFFIXES) or name == _OTHER_SOURCE_NAME + ): + self.blockers.add(_OTHER_SOURCE) + return + self.python_seen += 1 + if self.python_seen > _MAX_FILES: + self.over_limit = True + return + if counted: + self.has_counted_python = True + if "agents" in relative or name == "agents.py": + self.agents_in_counted_path = True + try: + data = _read_file( + os.path.join(directory, name), _MAX_TOTAL_BYTES - self.bytes_read + 1 + ) + except Exception: + if counted: + self.blockers.add(_FILE_SKIPPED) + return + self.bytes_read += len(data) + if self.bytes_read > _MAX_TOTAL_BYTES: + self.over_limit = True + return + if _has_long_line(data): + if counted: + self.blockers.add(_FILE_SKIPPED) + return + scan = _FileScan(self._shadowed, self._outside_names, self._work) + try: + failed = _scan_file(data, scan, read_uses=counted) + except _OverBudget: + self.over_limit = True + return + if not counted: + if scan.has_row_import: + self.blockers.add(_SIDE_FOLDER_FRAMEWORK) + return + if failed: + self.blockers.add(_FILE_UNREADABLE) + if scan.imports_outside: + self.blockers.add(_OUTSIDE_IMPORT) + file = "/".join((*relative, name)) + for (framework, strength), line in scan.found.items(): + self.found[(file, framework, strength)] = line + + +def _site_packages(package_dir: Path | None) -> list[str]: + if package_dir is None: + return [] + pattern = os.path.join( + glob.escape(os.fspath(package_dir)), ".venv", "lib", "python*", "site-packages" + ) + return sorted(path for path in glob.glob(pattern) if os.path.isdir(path)) + + +def _normalised(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def _installed(sites: list[str], package: str) -> list[str]: + """The `*.dist-info` directories of `package`, whatever their version.""" + wanted = _normalised(package) + matches = [] + for site in sites: + try: + names = sorted(os.listdir(site)) + except OSError: + continue + for name in names: + if ( + name.endswith(".dist-info") + and _normalised(name[: -len(".dist-info")].rpartition("-")[0]) == wanted + ): + matches.append(os.path.join(site, name)) + return matches + + +def _agents_rule( + walk: _FolderWalk, repo_root: Path | None, package_dir: Path | None +) -> None: + """`agents` is a common name for a customer's own package, so an import of it + only counts with evidence that the real one is meant.""" + if not any(framework == "openai_agents" for _, framework, _ in walk.found): + return + sites = _site_packages(package_dir) + if sites: + keep = bool(_installed(sites, "openai-agents")) + else: + keep = not walk.agents_in_counted_path and not ( + repo_root is not None + and ( + os.path.lexists(repo_root / "agents") + or os.path.lexists(repo_root / "agents.py") + ) + ) + if not keep: + walk.blockers.add(_AGENTS_AMBIGUOUS) + if not keep: + walk.found = { + key: line for key, line in walk.found.items() if key[1] != "openai_agents" + } + + +def folder_findings( + folder: Path, *, repo_root: Path | None = None, package_dir: Path | None = None +) -> FolderFindings: + folder = Path(folder) + blockers: set[str] = set() + walk = _FolderWalk( + _own_modules(folder, blockers), + _outside_names(folder, repo_root, blockers), + blockers, + ) + + def on_walk_error(_: OSError) -> None: + blockers.add(_FILE_SKIPPED) + + top = os.fspath(folder) + try: + for directory, directories, files in os.walk( + top, followlinks=False, onerror=on_walk_error + ): + # Sorted, so that a walk stopped by a limit stops at the same file. + directories.sort() + files.sort() + relative = tuple(directory[len(top) :].split(os.sep)[1:]) + if relative and any(name.endswith(".dist-info") for name in directories): + # An installed tree, not the customer's code. + directories[:] = [] + continue + directories[:] = _directories_to_enter( + directory, directories, at_top=not relative, blockers=blockers + ) + for name in files: + walk.take(directory, relative, name) + if walk.over_limit: + break + if walk.over_limit: + break + except (OSError, RecursionError): + # A tree deep enough to fail outside `onerror`. + blockers.add(_FILE_SKIPPED) + if not walk.has_counted_python: + blockers.add(_NO_PYTHON) + _agents_rule(walk, repo_root, package_dir) + findings = sorted( + ( + Finding(framework, strength, file, line) # type: ignore[arg-type] + for (file, framework, strength), line in walk.found.items() + ), + key=lambda f: (f.file, f.line, f.framework, f.strength), + ) + return FolderFindings(tuple(findings), frozenset(blockers), walk.over_limit) + + +def _decide(found: FolderFindings) -> tuple[str, str | None]: + """The result word, and the framework whose row it was read from.""" + if found.over_limit: + return "unknown", None + builds = {f.framework for f in found.findings if f.strength == "builds"} + named = builds & _NAMED_FRAMEWORKS + # A voice agent often drives a LangChain graph inside: the voice side is the agent. + if len(named & {"livekit", "pipecat"}) == 1: + named -= {"langchain", "langgraph", "deepagents"} + # Each is built on the next, so the higher one is what was chosen. + if "deepagents" in named: + named -= {"langgraph", "langchain"} + if "langgraph" in named: + named -= {"langchain"} + if len(named) == 1: + name = next(iter(named)) + return name, name + if named: + return "unknown", None + labels = builds - _NAMED_FRAMEWORKS + if len(labels) == 1: + return "other", next(iter(labels)) + if labels: + return "unknown", None + if any(f.strength == "model" for f in found.findings): + return "langchain", "langchain" + if any(f.strength == "import" for f in found.findings) or found.blockers: + return "unknown", None + return "none", None + + +def _version_from_metadata(dist_info: str) -> str | None: + data = _read_file(os.path.join(dist_info, "METADATA"), _METADATA_BYTES) + for line in data.split(b"\n"): + if line.startswith(b"Version:"): + value = line[len(b"Version:") :].strip().decode("ascii") + if len(value) <= _VERSION_MAX and _VERSION_PATTERN.fullmatch(value): + return value + return None + return None + + +def _read_version(row: _Row, package_dir: Path | None) -> str | None: + # A bad version is a missing detail, never a reason to doubt the name. + try: + sites = _site_packages(package_dir) + for package in row.packages: + matches = _installed(sites, package) + if matches: + return _version_from_metadata(matches[0]) if len(matches) == 1 else None + except Exception: + return None + return None + + +def detect_framework( + folder: Path | None, + *, + repo_root: Path | None = None, + package_dir: Path | None = None, +) -> dict[str, Any]: + try: + if folder is None or not Path(folder).is_dir(): + logger.info("framework detection: name=%s blockers=%s", "unknown", []) + return {"name": "unknown", "version": None} + found = folder_findings(folder, repo_root=repo_root, package_dir=package_dir) + name, framework = _decide(found) + result: dict[str, Any] = {"name": name, "version": None} + if framework is not None: + row = _ROW_BY_FRAMEWORK[framework] + result["version"] = _read_version(row, package_dir) + if name == "other": + result["label"] = framework + logger.info( + "framework detection: name=%s blockers=%s", name, sorted(found.blockers) + ) + return result + except Exception as exc: + logger.warning("framework detection failed: class=%s", type(exc).__name__) + return {"name": "unknown", "version": None} + + +def _contract_workdir(source: Path, bundle_dir: Path) -> Path | None: + """`runtime.workdir` from the sealed contract, when it names a real inner folder.""" + try: + contract = json.loads( + (bundle_dir / "contract.json").read_text(encoding="utf-8") + ) + runtime = contract.get("runtime") if isinstance(contract, dict) else None + workdir = runtime.get("workdir") if isinstance(runtime, dict) else None + if not isinstance(workdir, str) or not workdir or workdir == ".": + return None + candidate = source / workdir + resolved = candidate.resolve() + if resolved.is_relative_to(source.resolve()) and resolved.is_dir(): + return candidate + except (OSError, ValueError, RuntimeError): + return None + return None + + +def framework_folders_for( + job: Any, manifest: Any, *, source: Path, bundle_dir: Path, work_directory: Path +) -> FrameworkFolders: + runtime = manifest.runtime + control_service = runtime.control_service if runtime.kind == "process" else None + subfolder = job.source.subfolder + folder = source + if subfolder: + folder = agent_folder(source, subfolder) + else: + workdir = _contract_workdir(source, bundle_dir) + if workdir is not None: + folder = workdir + elif control_service: + for process in manifest.processes: + if process.name == control_service: + folder = bundle_dir / process.working_directory + break + package_dir = ( + work_directory / "build" / control_service if control_service else None + ) + return FrameworkFolders(folder, source, package_dir) diff --git a/src/fi/alk/harness/hosted_entrypoint.py b/src/fi/alk/harness/hosted_entrypoint.py index bbf274058..6910c3ffb 100644 --- a/src/fi/alk/harness/hosted_entrypoint.py +++ b/src/fi/alk/harness/hosted_entrypoint.py @@ -40,6 +40,7 @@ from .backends import GOOGLE_ADK_APPROVAL_ENV from .bundle_v2 import BundleV2Error, EnvironmentBundleV2, RuntimeKindV2, load_bundle_v2 from .compose_broker import BrokerError, ComposeBrokerClient +from .framework_detection import FrameworkFolders, framework_folders_for from .hosted_compose_bootstrap import initialize_hosted_compose from .source_folder import folder_failure from .world.observation import ObservationWorldFactory @@ -574,6 +575,29 @@ def _find_postgres_endpoint(runtime: EnvironmentRuntime) -> RuntimeEndpoint: ) +def _framework_folders( + job: HarnessJob, + manifest: EnvironmentBundleV2, + *, + source: Path, + bundle_dir: Path, + work_directory: Path, +) -> FrameworkFolders: + # Called outside the `try` around `build`: an exception here would end the run with no + # terminal event, and all that is at stake is a label. + try: + return framework_folders_for( + job, + manifest, + source=source, + bundle_dir=bundle_dir, + work_directory=work_directory, + ) + except Exception as exc: + logger.warning("framework folders failed: class=%s", type(exc).__name__) + return FrameworkFolders(source, source, None) + + def load_build_output(work_directory: Path) -> dict[str, Any]: """`write_build_output` (process_runtime.py) writes `/artifacts/build.json`. Read fresh each call — cheap, and the row counts are immutable after baseline freeze, so @@ -2674,7 +2698,15 @@ async def _fail( if isinstance(scenario_source, NotWiredScenarioSource) and bundle_has_scenarios( bundle_dir ): - scenario_source = BundleScenarioSource() + scenario_source = BundleScenarioSource( + framework_folders=_framework_folders( + job, + manifest, + source=source, + bundle_dir=bundle_dir, + work_directory=work_directory, + ) + ) try: scenarios = await scenario_source.build( job, diff --git a/src/fi/alk/harness/scenario_source.py b/src/fi/alk/harness/scenario_source.py index 03dd2edf5..5df3297a7 100644 --- a/src/fi/alk/harness/scenario_source.py +++ b/src/fi/alk/harness/scenario_source.py @@ -34,6 +34,7 @@ from . import outbound as ob from .catalogue import CATALOGUE, SubGoal from .contract import AgentContract +from .framework_detection import FrameworkFolders, detect_framework from .job import FailureDomain, SourceKind if TYPE_CHECKING: @@ -652,6 +653,27 @@ class BundleScenarioSource: always wins over this one. """ + def __init__(self, *, framework_folders: FrameworkFolders | None = None) -> None: + self._framework_folders = framework_folders + + async def _detect_framework(self) -> dict[str, Any] | None: + folders = self._framework_folders + if folders is None: + return None + # The framework is only a label: a failure here must never cost the document. + try: + return await asyncio.to_thread( + detect_framework, + folders.folder, + repo_root=folders.repo_root, + package_dir=folders.package_dir, + ) + except Exception as exc: + logger.warning( + "framework detection failed: class=%s", type(exc).__name__ + ) + return {"name": "unknown", "version": None} + async def build( self, job: Any, @@ -711,6 +733,7 @@ async def build( authored_agent = None source_kind = getattr(getattr(job, "source", None), "kind", None) if source_kind in (SourceKind.GITHUB, SourceKind.ARCHIVE): + framework = await self._detect_framework() try: contract_file = bundle_dir / "contract.json" contract = AgentContract.model_validate( @@ -725,7 +748,7 @@ async def build( authored_agent = { "sdk_version": _sdk_version, "agent": build_authored_agent( - contract, subfolder=built_from or None + contract, subfolder=built_from or None, framework=framework ), } except Exception as exc: diff --git a/src/fi/alk/providers/agent.py b/src/fi/alk/providers/agent.py index 5dc913cdc..e0cb43529 100644 --- a/src/fi/alk/providers/agent.py +++ b/src/fi/alk/providers/agent.py @@ -145,7 +145,10 @@ def __init__( def build_authored_agent( - document: Mapping[str, Any], *, subfolder: str | None = None + document: Mapping[str, Any], + *, + subfolder: str | None = None, + framework: Mapping[str, Any] | None = None, ) -> dict[str, Any]: contract = _redact( {key: value for key, value in document.items() if key != "base_environment"} @@ -181,7 +184,10 @@ def build_authored_agent( "direction_answer": None, "gaps": ["flow_not_described"], "instrumentation": contract.get("instrumentation") or "none", - "extension": {"contract": contract}, + "extension": { + "contract": contract, + **({"framework": dict(framework)} if framework is not None else {}), + }, } if subfolder: # The platform compares this to what it sent, so it is never trimmed or normalised. diff --git a/tests/harness/test_framework_detection.py b/tests/harness/test_framework_detection.py new file mode 100644 index 000000000..d93724357 --- /dev/null +++ b/tests/harness/test_framework_detection.py @@ -0,0 +1,1540 @@ +from __future__ import annotations + +import io +import json +import logging +import os +import shutil +import stat +import sys +import threading +import tokenize +import tracemalloc +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from fi.alk.harness import framework_detection as fd +from fi.alk.harness.framework_detection import ( + Finding, + FrameworkFolders, + detect_framework, + folder_findings, + framework_folders_for, +) +from fi.alk.harness.job import RepositorySource, SourceKind, SourceVisibility +from fi.alk.harness.source_folder import SubfolderNotFound + +EXAMPLES = Path(__file__).resolve().parents[2] / "examples" / "frameworks" +CREW = "from crewai import Crew\nCrew()\n" +PLAIN = "print('hello')\n" +LONG_LINE = "x = '" + "a" * 70_000 + "'\n" +ROT13 = "# coding: rot13\n" +UNKNOWN = {"name": "unknown", "version": None} +NONE = {"name": "none", "version": None} + + +def _write(folder: Path, files: dict[str, str]) -> None: + for relative, content in files.items(): + path = folder / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def _named(name: str, label: str | None = None) -> dict[str, Any]: + result: dict[str, Any] = {"name": name, "version": None} + if label is not None: + result["label"] = label + return result + + +def _dist(package_dir: Path, dist: str, metadata: bytes | None = None) -> Path: + folder = ( + package_dir + / ".venv" + / "lib" + / "python3.12" + / "site-packages" + / f"{dist}.dist-info" + ) + folder.mkdir(parents=True) + if metadata is not None: + (folder / "METADATA").write_bytes(metadata) + return folder + + +def _metadata(version: str) -> bytes: + return f"Metadata-Version: 2.1\nName: x\nVersion: {version}\n".encode() + + +@pytest.mark.parametrize( + ("example", "expected"), + [ + ("langchain_agent.py", _named("langchain")), + ("langgraph_agent.py", _named("langgraph")), + ("crewai_agent.py", _named("crewai")), + ("google_adk_agent.py", _named("google_adk")), + ("claude_agent_sdk_agent.py", _named("claude_agent_sdk")), + ("openai_agents_agent.py", _named("openai_agents")), + ("pydantic_ai_agent.py", _named("pydantic_ai")), + ("llamaindex_agent.py", _named("other", "llama_index")), + ("autogen_agent.py", _named("other", "autogen")), + ("semantic_kernel_agent.py", _named("other", "semantic_kernel")), + ("haystack_agent.py", _named("other", "haystack")), + ("agno_agent.py", _named("other", "agno")), + ("beeai_agent.py", _named("other", "beeai_framework")), + ("dspy_agent.py", _named("other", "dspy")), + ("smolagents_agent.py", _named("other", "smolagents")), + ("strands_agent.py", _named("other", "strands")), + ("acme_thirdparty_agent.py", NONE), + ], + ids=lambda value: value if isinstance(value, str) else "", +) +def test_examples_report_their_framework( + tmp_path: Path, example: str, expected: dict[str, Any] +) -> None: + shutil.copy(EXAMPLES / example, tmp_path / example) + + assert detect_framework(tmp_path) == expected + + +@pytest.mark.parametrize( + ("source", "name"), + [ + ( + "from livekit.agents import AgentSession, cli\n" + "cli.run_app(AgentSession())\n", + "livekit", + ), + ( + "from pipecat.pipeline.pipeline import Pipeline\nPipeline([])\n", + "pipecat", + ), + ( + "from deepagents import create_deep_agent\ncreate_deep_agent()\n", + "deepagents", + ), + ( + "from langgraph_supervisor import create_supervisor\ncreate_supervisor()\n", + "langgraph", + ), + ("from claude_code_sdk import query\nquery()\n", "claude_agent_sdk"), + ], + ids=["livekit", "pipecat", "deepagents", "langgraph_star", "claude_code_sdk"], +) +def test_every_framework_is_detected(tmp_path: Path, source: str, name: str) -> None: + _write(tmp_path, {"agent.py": source}) + + assert detect_framework(tmp_path) == _named(name) + + +LIVEKIT = "from livekit.agents import AgentSession\nAgentSession()\n" +PIPECAT = "from pipecat.pipeline.pipeline import Pipeline\nPipeline([])\n" +LANGGRAPH = "from langgraph.graph import StateGraph\nStateGraph(int)\n" +DEEP = "from deepagents import create_deep_agent\ncreate_deep_agent()\n" +LANGCHAIN = "from langchain.agents import create_agent\ncreate_agent()\n" +CHAT_OPENAI = "from langchain_openai import ChatOpenAI\nChatOpenAI()\n" +OPENAI_AGENT = "from agents import Agent\nAgent()\n" +AUTOGEN = "from autogen_agentchat.agents import AssistantAgent\nAssistantAgent()\n" +DSPY = "import dspy\ndspy.Predict('q')\n" +CREW_AGENT = "from crewai import Agent\n" +PYDANTIC_AGENT = "from pydantic_ai import Agent\n" + + +@pytest.mark.parametrize( + ("sources", "expected"), + [ + ((CREW, CHAT_OPENAI), _named("crewai")), + ((LIVEKIT, LANGGRAPH), _named("livekit")), + ((LIVEKIT, DEEP), _named("livekit")), + ((LIVEKIT, LANGCHAIN), _named("livekit")), + ((PIPECAT, LANGGRAPH), _named("pipecat")), + ((LANGGRAPH, LANGCHAIN), _named("langgraph")), + ((DEEP, LANGCHAIN), _named("deepagents")), + ((LIVEKIT, PIPECAT), UNKNOWN), + ((CREW, OPENAI_AGENT), UNKNOWN), + ((DEEP, LANGGRAPH), _named("deepagents")), + ((LANGGRAPH, CHAT_OPENAI), _named("langgraph")), + ((AUTOGEN, CHAT_OPENAI), _named("other", "autogen")), + ((AUTOGEN, DSPY), UNKNOWN), + (("from langchain_core.messages import HumanMessage\n",), UNKNOWN), + ( + ( + "from openai import OpenAI\nfrom anthropic import Anthropic\n" + "OpenAI()\nAnthropic()\n", + ), + NONE, + ), + ( + ( + "from langchain.schema import ChatMessage\n" + "from langchain.prompts import ChatPromptTemplate\n" + "from openai import OpenAI\n" + "ChatMessage()\nChatPromptTemplate()\nOpenAI()\n", + ), + UNKNOWN, + ), + ( + ("from langchain_openai import ChatOpenAI\nChatOpenAI().bind_tools([])\n",), + _named("langchain"), + ), + (("from crewai import *\nCrew()\n",), UNKNOWN), + ( + ("from importlib import import_module\nimport_module('crewai')\n",), + NONE, + ), + ((CREW_AGENT + PYDANTIC_AGENT + "Agent()\n",), UNKNOWN), + ((PYDANTIC_AGENT + CREW_AGENT + "Agent()\n",), UNKNOWN), + ], + ids=[ + "crew_and_model", + "livekit_and_langgraph", + "livekit_and_deepagents", + "livekit_and_langchain", + "pipecat_and_langgraph", + "langgraph_and_langchain", + "deepagents_and_langchain", + "livekit_and_pipecat", + "crew_and_openai_agents", + "deepagents_and_langgraph", + "langgraph_and_model", + "other_and_model", + "two_others", + "unused_import", + "direct_model_calls", + "chat_message_and_prompt", + "model_alone", + "star_import", + "dynamic_import", + "one_name_two_frameworks", + "one_name_two_frameworks_reversed", + ], +) +def test_decision_table( + tmp_path: Path, sources: tuple[str, ...], expected: dict[str, Any] +) -> None: + _write(tmp_path, {f"m{index}.py": s for index, s in enumerate(sources)}) + + assert detect_framework(tmp_path) == expected + + +CREW_IMPORT = ("crewai", "import") +CREW_BUILDS = ("crewai", "builds") +TYPE_ONLY_IMPORT = "from typing import TYPE_CHECKING\n{} from crewai import Crew\n" +LANGCHAIN_IMPORT = ("langchain", "import") +TYPE_CHECKING_LINE = "from typing import TYPE_CHECKING\n" + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + ( + "from langgraph.graph import StateGraph as SG\nSG(int)\n", + {("langgraph", "import"), ("langgraph", "builds")}, + ), + ( + 'from pydantic_ai import Agent\nAgent[int]("m")\n', + {("pydantic_ai", "import"), ("pydantic_ai", "builds")}, + ), + ( + "def f():\n from crewai import Crew\n Crew()\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ( + "try: import crewai\nexcept ImportError: crewai = None\ncrewai.Crew()\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ( + "from crewai import Crew\nclass C(Crew):\n pass\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ( + "from crewai import CrewBase\n@CrewBase\nclass C:\n pass\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ( + "from langgraph.func import entrypoint\n" + "@entrypoint()\ndef f(x):\n return x\n", + {("langgraph", "import"), ("langgraph", "builds")}, + ), + ( + TYPE_CHECKING_LINE + "if TYPE_CHECKING:\n from crewai import Crew\n", + set(), + ), + ( + "import typing\nif typing.TYPE_CHECKING:\n from crewai import Crew\n", + set(), + ), + ( + TYPE_CHECKING_LINE + + "if TYPE_CHECKING:\n import os\nelse:\n from crewai import Crew\n" + + "Crew()\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ("from .crewai import Crew\nCrew()\n", set()), + ("from . import crewai\ncrewai.Crew()\n", set()), + ("from ... import crewai\ncrewai.Crew()\n", set()), + ("from ...crewai import Crew\nCrew()\n", set()), + ( + "from crewai import (\n Agent,\n Crew as C,\n)\nC()\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ("import os, crewai\ncrewai.Crew()\n", {CREW_IMPORT, CREW_BUILDS}), + ("import os; import crewai\ncrewai.Crew()\n", {CREW_IMPORT, CREW_BUILDS}), + ( + "import google.adk\ngoogle.adk.agents.Agent()\n", + {("google_adk", "import"), ("google_adk", "builds")}, + ), + ("from crewai import Crew\ndef Crew(x):\n return x\n", {CREW_IMPORT}), + ("from crewai import Crew\nclass Crew:\n pass\n", {CREW_IMPORT}), + ('from crewai import Crew\n# Crew()\nx = "Crew()"\n', {CREW_IMPORT}), + ("from crewai import Crew\nf(Crew=1)\n", {CREW_IMPORT}), + ("from crewai import Crew\nx = a @ Crew\n", {CREW_IMPORT}), + ( + "from crewai import Crew\nclass C(Base, metaclass=Crew):\n pass\n", + {CREW_IMPORT}, + ), + ("import dspy\nclass Q(dspy.Module):\n pass\n", {("dspy", "import")}), + ( + "from langchain_openai import AzureChatOpenAI\nAzureChatOpenAI()\n", + {("langchain", "import"), ("langchain", "model")}, + ), + ( + 'from langchain.chat_models import init_chat_model\ninit_chat_model("m")\n', + {("langchain", "import"), ("langchain", "model")}, + ), + ('from crewai import Crew\nx = f"{Crew()}"\n', {CREW_IMPORT}), + ( + "print(f'{x}')\nfrom crewai import Crew\nCrew()\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ("from crewai import Crew\nself.Crew()\n", {CREW_IMPORT}), + ( + "import google.adk.agents as g\ng.LlmAgent()\n", + {("google_adk", "import"), ("google_adk", "builds")}, + ), + ("from langchain_core.x import ChatFoo\nChatFoo()\n", {LANGCHAIN_IMPORT}), + ( + "from langchain_core.outputs import ChatGeneration\n" + "from langchain.x import ChatGeneration as C\nC()\n", + {LANGCHAIN_IMPORT}, + ), + ("from langchain.x import ChatResult\nChatResult()\n", {LANGCHAIN_IMPORT}), + ("def f():\n yield from crewai\n", set()), + ("raise E from crewai\n", set()), + ( + "from crewai import Crew\nclass C[T](Crew):\n pass\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ( + "from crewai import Crew\nclass C[T: (A, B)](Crew):\n pass\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ("from crewai import Crew\nclass C: x = (Crew)\n", {CREW_IMPORT}), + ( + TYPE_ONLY_IMPORT.format("if TYPE_CHECKING: # pragma: no cover\n"), + set(), + ), + ( + TYPE_ONLY_IMPORT.format( + "try:\n import os\nexcept E:\n pass\nif TYPE_CHECKING:\n" + ), + set(), + ), + ( + TYPE_ONLY_IMPORT.format( + "if TYPE_CHECKING:\n if x:\n import os\n" + ), + set(), + ), + ( + "from crewai import Crew\nclass C(\n Crew,\n):\n pass\n", + {CREW_IMPORT, CREW_BUILDS}, + ), + ("from crewai import Crew\nclass C(A): x = (Crew)\n", {CREW_IMPORT}), + ("from crewai import Crew\nclass C(make(Crew)):\n pass\n", {CREW_IMPORT}), + ("from crewai import Crew\nx = f\"{f'{a}'} {Crew()}\"\n", {CREW_IMPORT}), + ], + ids=[ + "alias", + "subscript_then_call", + "import_inside_function", + "import_after_a_block_colon", + "class_base", + "decorator", + "decorator_with_call", + "type_checking_block", + "dotted_type_checking_block", + "type_checking_else", + "relative_module", + "relative_from_dot", + "relative_from_ellipsis", + "relative_ellipsis_module", + "parenthesised", + "comma_import", + "semicolon_import", + "dotted_use", + "def_name", + "class_name", + "comment_and_string", + "keyword_name", + "matrix_multiply", + "metaclass_value", + "other_row_base_alone", + "azure_model", + "init_chat_model", + "inside_fstring", + "after_fstring", + "method_of_another_object", + "dotted_alias", + "langchain_core_model_excluded", + "chat_generation_excluded", + "chat_result_excluded", + "yield_from", + "raise_from", + "generic_class_base", + "generic_class_constraint_brackets", + "class_body_on_one_line", + "type_checking_with_a_comment", + "type_checking_after_a_block", + "type_checking_with_a_nested_block", + "base_on_its_own_line", + "bracket_in_a_one_line_class_body", + "call_inside_the_bases", + "after_nested_fstring", + ], +) +def test_findings_rules( + tmp_path: Path, source: str, expected: set[tuple[str, str]] +) -> None: + _write(tmp_path, {"agent.py": source}) + + found = folder_findings(tmp_path) + + assert {(f.framework, f.strength) for f in found.findings} == expected + + +def test_one_finding_per_file_framework_strength(tmp_path: Path) -> None: + _write(tmp_path, {"agent.py": "from crewai import Crew\nCrew()\nCrew()\nCrew()\n"}) + + assert folder_findings(tmp_path).findings == ( + Finding("crewai", "import", "agent.py", 1), + Finding("crewai", "builds", "agent.py", 2), + ) + + +def test_findings_same_when_a_limit_stops_the_walk( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write(tmp_path, {"a.py": CREW, "b.py": PIPECAT, "z/c.py": LANGGRAPH}) + monkeypatch.setattr(fd, "_MAX_FILES", 1) + in_order = folder_findings(tmp_path) + real_walk = os.walk + + def reversed_walk(*args: Any, **kwargs: Any) -> Any: + for directory, directories, files in real_walk(*args, **kwargs): + directories.reverse() + files.reverse() + yield directory, directories, files + + monkeypatch.setattr(fd.os, "walk", reversed_walk) + + assert in_order.over_limit + assert folder_findings(tmp_path) == in_order + assert [f.file for f in in_order.findings] == ["a.py", "a.py"] + + +@pytest.mark.parametrize( + ("files", "expected"), + [ + ({"main.py": PLAIN, "build/x.py": CREW}, NONE), + ({"main.py": PLAIN, "dist/x.py": CREW}, NONE), + ({"main.py": PLAIN, "env/x.py": CREW}, NONE), + ({"pkg/build/x.py": CREW}, _named("crewai")), + ({"pkg/dist/x.py": CREW}, _named("crewai")), + ({"pkg/env/x.py": CREW}, _named("crewai")), + ({"main.py": PLAIN, "pkg/node_modules/x.py": CREW}, NONE), + ({"main.py": PLAIN, "pkg/site-packages/x.py": CREW}, NONE), + ( + { + "main.py": PLAIN, + "lib/x.py": CREW, + "lib/crewai-1.0.dist-info/METADATA": "", + }, + NONE, + ), + ( + { + "main.py": CREW, + "crewai/__init__.py": "", + "crewai-1.0.dist-info/METADATA": "", + }, + _named("crewai"), + ), + ( + {"main.py": PLAIN, "crewai-1.0.dist-info/METADATA": ""}, + UNKNOWN, + ), + ], + ids=[ + "build_at_top", + "dist_at_top", + "env_at_top", + "build_deeper", + "dist_deeper", + "env_deeper", + "node_modules_at_depth", + "site_packages_at_depth", + "dist_info_child_stops_descent", + "pip_install_target_not_shadowed", + "dist_info_at_top_blocks_none", + ], +) +def test_walk_rules( + tmp_path: Path, files: dict[str, str], expected: dict[str, Any] +) -> None: + _write(tmp_path, files) + + assert detect_framework(tmp_path) == expected + + +def test_walk_rules_unreadable_directory_blocks_none(tmp_path: Path) -> None: + if os.geteuid() == 0: + pytest.skip("root reads a mode-000 directory") + _write(tmp_path, {"main.py": PLAIN, "locked/x.py": CREW}) + locked = tmp_path / "locked" + locked.chmod(0) + try: + found = folder_findings(tmp_path) + result = detect_framework(tmp_path) + finally: + locked.chmod(stat.S_IRWXU) + + assert fd._FILE_SKIPPED in found.blockers + assert result == UNKNOWN + + +def test_walk_rules_symlinked_directory_blocks_none(tmp_path: Path) -> None: + folder = tmp_path / "agent" + _write(folder, {"main.py": PLAIN}) + _write(tmp_path, {"elsewhere/x.py": CREW}) + (folder / "link").symlink_to(tmp_path / "elsewhere", target_is_directory=True) + + assert fd._FILE_SKIPPED in folder_findings(folder).blockers + assert detect_framework(folder) == UNKNOWN + + +def test_byte_order_mark_and_latin1_files_read(tmp_path: Path) -> None: + (tmp_path / "bom.py").write_bytes(b"\xef\xbb\xbf" + CREW.encode()) + assert detect_framework(tmp_path) == _named("crewai") + + (tmp_path / "bom.py").unlink() + (tmp_path / "latin.py").write_bytes( + b"# -*- coding: latin-1 -*-\n# caf\xe9\n" + CREW.encode() + ) + assert detect_framework(tmp_path) == _named("crewai") + + +def test_long_line_in_a_side_file_is_not_scanned(tmp_path: Path) -> None: + _write( + tmp_path, + {"main.py": PLAIN, "tests/x.py": "from crewai import Crew\n" + LONG_LINE}, + ) + + assert detect_framework(tmp_path) == NONE + + +def test_long_line_blocks_none_only(tmp_path: Path) -> None: + _write(tmp_path, {"main.py": PLAIN, "long.py": LONG_LINE}) + assert detect_framework(tmp_path) == UNKNOWN + + _write(tmp_path, {"crew.py": CREW}) + assert detect_framework(tmp_path) == _named("crewai") + + +@pytest.mark.parametrize( + ("content", "skipped"), + [ + (CREW + LONG_LINE, True), + ("#" * fd._MAX_LINE_BYTES + "\n", False), + ("#" * (fd._MAX_LINE_BYTES + 1) + "\n", True), + (PLAIN + "#" * (fd._MAX_LINE_BYTES + 1), True), + ], + ids=[ + "after_a_short_line", + "exactly_at_the_limit", + "one_over", + "last_with_no_newline", + ], +) +def test_long_line_on_any_line(tmp_path: Path, content: str, skipped: bool) -> None: + _write(tmp_path, {"main.py": content}) + + found = folder_findings(tmp_path) + + assert (fd._FILE_SKIPPED in found.blockers) is skipped + assert detect_framework(tmp_path) == (UNKNOWN if skipped else NONE) + + +def _in_thread(call: Any) -> Any: + results: list[Any] = [] + thread = threading.Thread(target=lambda: results.append(call()), daemon=True) + thread.start() + thread.join(10) + assert not thread.is_alive(), "detection blocked on a named pipe" + return results[0] + + +def test_named_pipes_never_block(tmp_path: Path) -> None: + plain = tmp_path / "plain" + _write(plain, {"main.py": PLAIN}) + os.mkfifo(plain / "x.py") + assert _in_thread(lambda: detect_framework(plain)) == UNKNOWN + + crew = tmp_path / "crew" + _write(crew, {"main.py": CREW}) + package_dir = tmp_path / "package" + os.mkfifo(_dist(package_dir, "crewai-1.0") / "METADATA") + assert _in_thread( + lambda: detect_framework(crew, package_dir=package_dir) + ) == _named("crewai") + + +def test_symlinked_python_is_not_followed(tmp_path: Path) -> None: + folder = tmp_path / "agent" + folder.mkdir() + _write(tmp_path, {"elsewhere.py": CREW}) + (folder / "x.py").symlink_to(tmp_path / "elsewhere.py") + + assert detect_framework(folder) == UNKNOWN + + +@pytest.mark.parametrize( + ("files", "expected"), + [ + ({"crew.py": CREW, "bad.py": ROT13}, _named("crewai")), + ({"main.py": PLAIN, "bad.py": ROT13}, UNKNOWN), + ({"agent.py": CREW + "x = '''\n"}, _named("crewai")), + ({"agent.py": "x = (\n" + CREW}, _named("crewai")), + ], + ids=[ + "bad_neighbour_keeps_crew", + "bad_neighbour_blocks_none", + "error_after_use", + "import_inside_an_open_bracket", + ], +) +def test_failure_scanning_a_file_keeps_what_was_found( + tmp_path: Path, files: dict[str, str], expected: dict[str, Any] +) -> None: + _write(tmp_path, files) + + assert fd._FILE_UNREADABLE in folder_findings(tmp_path).blockers + assert detect_framework(tmp_path) == expected + + +def test_failure_scanning_a_file_keeps_what_was_found_any_exception( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write(tmp_path, {"agent.py": CREW}) + real = fd._visible_tokens + + def fails_after_the_use(data: bytes, work: Any) -> Any: + yield from real(data, work) + raise SystemError("from the tokenizer") + + monkeypatch.setattr(fd, "_visible_tokens", fails_after_the_use) + + found = folder_findings(tmp_path) + + assert fd._FILE_UNREADABLE in found.blockers + assert CREW_BUILDS in {(f.framework, f.strength) for f in found.findings} + + +def test_too_many_aliases_is_unreadable( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(fd, "_MAX_ALIASES", 1) + _write(tmp_path, {"agent.py": "from crewai import Crew, Agent\n"}) + + assert fd._FILE_UNREADABLE in folder_findings(tmp_path).blockers + + +@pytest.mark.parametrize( + ("constant", "value"), + [("_MAX_FILES", 1), ("_MAX_TOTAL_BYTES", 10)], +) +def test_total_limits_give_unknown( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, constant: str, value: int +) -> None: + _write(tmp_path, {"a.py": CREW, "b.py": CREW}) + monkeypatch.setattr(fd, constant, value) + + assert folder_findings(tmp_path).over_limit + assert detect_framework(tmp_path) == UNKNOWN + + +@pytest.mark.parametrize( + ("constant", "value", "side_file"), + [("_MAX_FILES", 1, PLAIN), ("_MAX_TOTAL_BYTES", 20, "#" * 50 + "\n")], + ids=["files", "bytes"], +) +def test_total_limits_count_side_files( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + constant: str, + value: int, + side_file: str, +) -> None: + _write(tmp_path, {"main.py": PLAIN, "tests/a.py": side_file}) + monkeypatch.setattr(fd, constant, value) + + assert folder_findings(tmp_path).over_limit + + +def test_limits_have_their_real_values() -> None: + assert ( + fd._MAX_LINE_BYTES, + fd._MAX_TOTAL_BYTES, + fd._MAX_FILES, + fd._MAX_WORK, + fd._WORK_PER_TARGET, + fd._MAX_ALIASES, + fd._MAX_DOTTED_PARTS, + fd._METADATA_BYTES, + fd._VERSION_MAX, + ) == ( + 65_536, + 67_108_864, + 5_000, + 100_000_000_000, + 2_000, + 10_000, + 64, + 65_536, + 64, + ) + + +def test_read_stops_at_the_bytes_left( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write(tmp_path, {"a.py": "x=1\n", "b.py": "y=2\n"}) + limits: list[int] = [] + real = fd._read_file + + def spy(path: str, limit: int) -> bytes: + limits.append(limit) + return real(path, limit) + + monkeypatch.setattr(fd, "_read_file", spy) + + folder_findings(tmp_path) + + assert limits == [fd._MAX_TOTAL_BYTES + 1, fd._MAX_TOTAL_BYTES - 4 + 1] + + +def test_tokenize_warnings_are_silenced( + tmp_path: Path, capfd: pytest.CaptureFixture[str], recwarn: pytest.WarningsRecorder +) -> None: + _write(tmp_path, {"a.py": "x = f'\\{a}'\n" * 3}) + + detect_framework(tmp_path) + + assert not recwarn.list + assert "Warning" not in capfd.readouterr().err + + +def _token_work(source: str) -> int: + tokens = tokenize.tokenize(io.BytesIO(source.encode()).readline) + return sum(len(token.line) for token in tokens) + + +# Two reads, and two walks of its one target: the name in the import, then the call. +CREW_WORK = 2 * _token_work(CREW) + 2 * fd._WORK_PER_TARGET +# Comments and what's inside an f-string are never seen by the scan, but cost the same. +UNSEEN = "# note\nx = f'{a}'\n" + CREW +UNSEEN_WORK = 2 * _token_work(UNSEEN) + 2 * fd._WORK_PER_TARGET + + +@pytest.mark.parametrize( + ("files", "work"), + [ + ({"a.py": CREW}, CREW_WORK), + ({"a.py": CREW, "b.py": CREW}, 2 * CREW_WORK), + ({"main.py": CREW, "tests/a.py": PLAIN}, CREW_WORK + _token_work(PLAIN)), + ({"a.py": UNSEEN}, UNSEEN_WORK), + ], + ids=[ + "one_target", + "shared_across_files", + "side_file_read_once", + "tokens_the_scan_never_sees", + ], +) +def test_work_limit_edge( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + files: dict[str, str], + work: int, +) -> None: + _write(tmp_path, files) + + monkeypatch.setattr(fd, "_MAX_WORK", work) + assert detect_framework(tmp_path) == _named("crewai") + + monkeypatch.setattr(fd, "_MAX_WORK", work - 1) + assert folder_findings(tmp_path).over_limit + assert detect_framework(tmp_path) == UNKNOWN + + +def test_work_limit_counts_every_target_of_a_name( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + imports = "".join(f"import crewai.m{index} as C\n" for index in range(10_000)) + source = imports + "C()\n" * 100 + limit = 1_000_000_000 + assert 2 * _token_work(source) < limit + _write(tmp_path, {"agent.py": source}) + monkeypatch.setattr(fd, "_MAX_WORK", limit) + + found = folder_findings(tmp_path) + + assert found.over_limit + assert fd._FILE_UNREADABLE not in found.blockers + assert detect_framework(tmp_path) == UNKNOWN + + +def test_work_limit_counts_long_lines( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + long_line = "x = [" + "a," * 30_000 + "]\n" + assert len(long_line) <= fd._MAX_LINE_BYTES + _write(tmp_path, {"agent.py": CREW + long_line * 2}) + monkeypatch.setattr(fd, "_MAX_WORK", 10_000_000) + + found = folder_findings(tmp_path) + + assert found.over_limit + assert detect_framework(tmp_path) == UNKNOWN + + +def test_bytes_exactly_at_the_limit_are_not_over( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write(tmp_path, {"a.py": CREW}) + monkeypatch.setattr(fd, "_MAX_TOTAL_BYTES", len(CREW)) + + assert not folder_findings(tmp_path).over_limit + + +LONG_PART = b"p" * 65_000 + + +def test_names_under_one_long_module_share_its_memory(tmp_path: Path) -> None: + module = b"from crewai.\\\n" + (LONG_PART + b".\\\n") * 62 + LONG_PART + names = b"".join(b"a%d,\n" % index for index in range(100)) + (tmp_path / "agent.py").write_bytes(module + b" import (\n" + names + b")\n") + + tracemalloc.start() + try: + found = folder_findings(tmp_path) + peak = tracemalloc.get_traced_memory()[1] + finally: + tracemalloc.stop() + + assert peak < 50 * 1024 * 1024 + assert {(f.framework, f.strength) for f in found.findings} == {CREW_IMPORT} + + +def _aliases(source: str) -> dict[str, set[tuple[str, ...]]]: + scan = fd._FileScan(frozenset(), frozenset(), fd._Work()) + scan.read_imports( + fd._outside_type_checking(fd._visible_tokens(source.encode(), scan.work)) + ) + return scan.aliases + + +def test_a_target_keeps_at_most_the_part_limit() -> None: + aliases = _aliases("import crewai." + "a." * 200 + "z as C\n") + + assert [len(target) for target in aliases["C"]] == [fd._MAX_DOTTED_PARTS] + + +def test_plain_dotted_import_maps_the_top_name_to_the_top_name() -> None: + assert _aliases("import crewai.tools\n") == {"crewai": {("crewai",)}} + + +def test_use_inside_a_type_checking_block_does_not_count(tmp_path: Path) -> None: + _write( + tmp_path, + {"agent.py": "from crewai import Crew\nif TYPE_CHECKING:\n Crew()\n"}, + ) + + found = folder_findings(tmp_path) + + assert {(f.framework, f.strength) for f in found.findings} == {CREW_IMPORT} + + +def test_a_use_of_a_long_target_allocates_nothing_of_its_size() -> None: + data = b"import crewai.\\\n" + (LONG_PART + b".\\\n") * 63 + LONG_PART + b" as C\n" + scan = fd._FileScan(frozenset(), frozenset(), fd._Work()) + scan.read_imports(fd._outside_type_checking(fd._visible_tokens(data, scan.work))) + + tracemalloc.start() + try: + scan._resolve(["C"], "call", 1) + peak = tracemalloc.get_traced_memory()[1] + finally: + tracemalloc.stop() + + assert peak < 64 * 1024 + + +BLOCKER_CASES: list[tuple[str, dict[str, str], str, str]] = [ + ( + fd._FILE_SKIPPED, + {"agent/main.py": PLAIN, "agent/long.py": LONG_LINE}, + "agent", + "", + ), + (fd._FILE_UNREADABLE, {"agent/main.py": PLAIN, "agent/bad.py": ROT13}, "agent", ""), + ( + fd._AGENTS_AMBIGUOUS, + {"agent/main.py": OPENAI_AGENT, "agent/src/agents/x.py": PLAIN}, + "agent", + "", + ), + ( + fd._SIDE_FOLDER_FRAMEWORK, + {"agent/main.py": PLAIN, "agent/tests/x.py": CREW}, + "agent", + "", + ), + ( + fd._OUTSIDE_IMPORT, + {"agent/main.py": "import shared\n", "shared.py": PLAIN}, + "agent", + ".", + ), + ( + fd._OUTSIDE_IMPORT, + {"agent/main.py": "from shared import x\nimport os, tools\n", "tools/y.py": ""}, + "agent", + ".", + ), + (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/ui.ts": ""}, "agent", ""), + (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/ui.tsx": ""}, "agent", ""), + (fd._OTHER_SOURCE, {"agent/main.py": PLAIN, "agent/n.ipynb": ""}, "agent", ""), + ( + fd._OTHER_SOURCE, + {"agent/main.py": PLAIN, "agent/root_agent.yaml": ""}, + "agent", + "", + ), + (fd._NO_PYTHON, {"agent/README.md": ""}, "agent", ""), +] + + +@pytest.mark.parametrize( + ("blocker", "files", "folder", "repo_root"), + BLOCKER_CASES, + ids=[ + "file_skipped", + "file_unreadable", + "agents_ambiguous", + "side_folder_framework", + "outside_import", + "outside_import_from_and_comma", + "other_source_ts", + "other_source_tsx", + "other_source_ipynb", + "other_source_root_agent", + "no_python", + ], +) +def test_blockers_stop_none( + tmp_path: Path, blocker: str, files: dict[str, str], folder: str, repo_root: str +) -> None: + _write(tmp_path, files) + root = tmp_path if repo_root else None + + assert blocker in folder_findings(tmp_path / folder, repo_root=root).blockers + assert detect_framework(tmp_path / folder, repo_root=root) == UNKNOWN + + # A blocker stops `none` and nothing else. + _write(tmp_path, {f"{folder}/crew.py": CREW}) + assert detect_framework(tmp_path / folder, repo_root=root) == _named("crewai") + + +@pytest.mark.parametrize( + ("folder", "files", "blocked"), + [ + ("repo/agent", {"repo/agent/main.py": "import agent\n"}, False), + ("repo/a/b", {"repo/a/b/main.py": "import a\n"}, False), + ("repo/agent", {"repo/agent/main.py": "from . import shared\n"}, False), + ( + "repo/agent", + {"repo/agent/main.py": "import shared\n", "repo/agent/shared.py": ""}, + False, + ), + ("copy", {"copy/main.py": "import agent\n", "repo/agent/x.py": ""}, True), + ], + ids=[ + "folder_itself", + "folder_on_the_path", + "relative", + "under_the_folder", + "folder_outside_the_root_excludes_nothing", + ], +) +def test_blockers_stop_none_outside_import_scope( + tmp_path: Path, folder: str, files: dict[str, str], blocked: bool +) -> None: + _write(tmp_path, files) + (tmp_path / "repo" / "shared.py").write_text(PLAIN, encoding="utf-8") + (tmp_path / "repo" / "a").mkdir(exist_ok=True) + (tmp_path / "repo" / "agent").mkdir(exist_ok=True) + + found = folder_findings(tmp_path / folder, repo_root=tmp_path / "repo") + + assert (fd._OUTSIDE_IMPORT in found.blockers) is blocked + + +def test_blockers_stop_none_no_outside_import_when_folder_is_the_root( + tmp_path: Path, +) -> None: + _write(tmp_path, {"main.py": "import shared\n", "shared.py": PLAIN}) + + assert detect_framework(tmp_path, repo_root=tmp_path) == NONE + + +SIDE_WORDS = [ + "tests", + "test", + "examples", + "example", + "samples", + "sample", + "demos", + "demo", + "docs", + "doc", + "evals", + "eval", + "scripts", + "legacy", + "notebooks", + "vendor", + "vendored", + "third_party", + "external", + "contrib", + "experiments", + "benchmarks", + "archive", + ".hidden", +] +SIDE_PATHS = [f"{word}/crew.py" for word in SIDE_WORDS] + [ + "test_x.py", + "x_test.py", + "conftest.py", + "pkg/tests/crew.py", + ".x.py", +] + + +@pytest.mark.parametrize("path", SIDE_PATHS) +def test_side_folders_do_not_decide(tmp_path: Path, path: str) -> None: + _write(tmp_path, {"main.py": PLAIN, path: CREW}) + + found = folder_findings(tmp_path) + + assert found.findings == () + assert found.blockers == {fd._SIDE_FOLDER_FRAMEWORK} + assert detect_framework(tmp_path) == UNKNOWN + + +def test_side_folder_as_the_agent_folder_counts(tmp_path: Path) -> None: + _write(tmp_path, {"examples/x/main.py": CREW}) + + assert detect_framework(tmp_path / "examples" / "x") == _named("crewai") + + +@pytest.mark.parametrize("content", [LONG_LINE, ROT13], ids=["long_line", "rot13"]) +def test_problem_in_a_side_file_is_no_blocker(tmp_path: Path, content: str) -> None: + _write(tmp_path, {"main.py": PLAIN, "tests/bad.py": content}) + + assert detect_framework(tmp_path) == NONE + + +def test_agents_folder_beside_plain_code_is_none(tmp_path: Path) -> None: + _write(tmp_path, {"main.py": PLAIN, "agents/x.py": PLAIN}) + + assert detect_framework(tmp_path) == NONE + + +@pytest.mark.parametrize( + ("files", "installed", "repo_root", "expected"), + [ + ({"main.py": OPENAI_AGENT}, "openai_agents-0.1", None, "openai_agents"), + ({"main.py": OPENAI_AGENT}, "requests-2.0", None, "none"), + ({"main.py": OPENAI_AGENT}, None, None, "openai_agents"), + ({"main.py": OPENAI_AGENT, "src/agents/x.py": PLAIN}, None, None, "unknown"), + ({"main.py": OPENAI_AGENT, "agents.py": PLAIN}, None, None, "unknown"), + ( + {"main.py": OPENAI_AGENT, "agents.py": PLAIN}, + "openai_agents-0.1", + None, + "openai_agents", + ), + ({"main.py": OPENAI_AGENT}, None, "with_agents", "unknown"), + ], + ids=[ + "installed", + "other_packages_installed", + "no_tree_no_folder", + "no_tree_own_folder", + "no_tree_own_module", + "own_module_with_package_installed", + "no_tree_folder_under_repo_root", + ], +) +def test_agents_import_rules( + tmp_path: Path, + files: dict[str, str], + installed: str | None, + repo_root: str | None, + expected: str, +) -> None: + folder = tmp_path / "repo" / "agent" + _write(folder, files) + package_dir = None + if installed is not None: + package_dir = tmp_path / "package" + _dist(package_dir, installed) + root = None + if repo_root is not None: + root = tmp_path / "repo" + (root / "agents").mkdir() + + result = detect_framework(folder, repo_root=root, package_dir=package_dir) + + assert result["name"] == expected + + +def test_agents_module_under_repo_root_is_ambiguous(tmp_path: Path) -> None: + _write(tmp_path, {"agent/main.py": OPENAI_AGENT, "agents.py": PLAIN}) + + assert detect_framework(tmp_path / "agent", repo_root=tmp_path) == UNKNOWN + + +def test_folder_with_no_init_does_not_shadow(tmp_path: Path) -> None: + _write(tmp_path, {"main.py": CREW, "crewai/x.py": PLAIN}) + + assert detect_framework(tmp_path) == _named("crewai") + + +@pytest.mark.parametrize( + "files", + [ + {"main.py": CREW, "crewai.py": PLAIN}, + { + "main.py": "from langchain.agents import create_agent\ncreate_agent()\n", + "langchain/__init__.py": "", + }, + ], + ids=["module", "package"], +) +def test_own_module_shadows_framework(tmp_path: Path, files: dict[str, str]) -> None: + _write(tmp_path, files) + + assert detect_framework(tmp_path) == NONE + + +def test_missing_folder_gives_unknown(tmp_path: Path) -> None: + assert detect_framework(None) == UNKNOWN + assert detect_framework(tmp_path / "missing") == UNKNOWN + + +def test_detector_failure_gives_unknown( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + secret = "/customer/private/path" + + def _raise(*_: Any, **__: Any) -> Any: + raise RuntimeError(secret) + + monkeypatch.setattr(fd, "folder_findings", _raise) + + with caplog.at_level(logging.INFO, logger=fd.__name__): + result = detect_framework(tmp_path) + + assert result == UNKNOWN + assert "RuntimeError" in caplog.text + assert secret not in caplog.text + + +@pytest.mark.parametrize( + "failure", [MemoryError(), RecursionError()], ids=["memory", "recursion"] +) +def test_detector_failure_gives_unknown_on_resource_errors( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: Exception +) -> None: + def _raise(*_: Any, **__: Any) -> Any: + raise failure + + monkeypatch.setattr(fd, "folder_findings", _raise) + + assert detect_framework(tmp_path) == UNKNOWN + + +def test_never_imports_or_runs_customer_code(tmp_path: Path) -> None: + marker = tmp_path / "ran" + source = ( + f"from pathlib import Path\nPath({str(marker)!r}).write_text('x')\n" + "from crewai import Crew\nCrew()\n" + ) + folder = tmp_path / "agent" + _write(folder, {"customer_module_never_imported.py": source}) + + assert detect_framework(folder) == _named("crewai") + + assert not marker.exists() + assert "customer_module_never_imported" not in sys.modules + assert "crewai" not in sys.modules + + +LAYERED = DEEP + LANGGRAPH +PYDANTIC_BUILDS = "from pydantic_ai import Agent\nAgent()\n" + + +@pytest.mark.parametrize( + ("source", "dists", "expected"), + [ + ( + "from pydantic_ai import Agent\nAgent()\n", + [("pydantic_ai_slim-1.2.3", "1.2.3")], + "1.2.3", + ), + ( + "from pydantic_ai import Agent\nAgent()\n", + [("pydantic_ai-2.0", "2.0"), ("pydantic_ai_slim-1.2.3", "1.2.3")], + "2.0", + ), + (LIVEKIT, [("livekit_agents-1.8.8", "1.8.8")], "1.8.8"), + (LIVEKIT, [("livekit-1.1.20", "1.1.20")], None), + (CREW, [("crewai-1.0", "1.0"), ("crewai-2.0", "2.0")], None), + (LAYERED, [("deepagents-0.2", "0.2"), ("langgraph-1.0", "1.0")], "0.2"), + ( + AUTOGEN, + [("autogen_agentchat-0.4.1", "0.4.1"), ("pyautogen-0.2", "0.2")], + "0.4.1", + ), + ( + CREW, + [("crewai-1.0rc1.post2.dev3+local.1", "1.0rc1.post2.dev3+local.1")], + "1.0rc1.post2.dev3+local.1", + ), + (PYDANTIC_BUILDS, [("PyDantic_AI-1.0", "1.0")], "1.0"), + (PYDANTIC_BUILDS, [("pydantic.ai-1.0", "1.0")], "1.0"), + (PYDANTIC_BUILDS, [("pydantic_ai-1.0", "1.0RC1")], "1.0RC1"), + ], + ids=[ + "second_package_of_the_row", + "first_package_wins", + "livekit_agents", + "livekit_alone_is_not_the_row", + "two_versions", + "layered_reads_the_chosen_row", + "other_row", + "full_version_shape", + "name_case_ignored", + "name_dots_normalised", + "version_case_ignored", + ], +) +def test_version_read( + tmp_path: Path, source: str, dists: list[tuple[str, str]], expected: str | None +) -> None: + folder = tmp_path / "agent" + _write(folder, {"main.py": source}) + package_dir = tmp_path / "package" + for dist, version in dists: + _dist(package_dir, dist, _metadata(version)) + + assert detect_framework(folder, package_dir=package_dir)["version"] == expected + + +@pytest.mark.parametrize( + ("python", "metadata"), + [("python3.13", b"Version: 1.0\n"), ("python3.12", b"Version: 1.0\r\n")], + ids=["any_python_folder", "crlf_line"], +) +def test_version_read_real_shapes(tmp_path: Path, python: str, metadata: bytes) -> None: + folder = tmp_path / "agent" + _write(folder, {"main.py": CREW}) + dist_info = ( + tmp_path / "pkg/.venv/lib" / python / "site-packages/crewai-1.0.dist-info" + ) + dist_info.mkdir(parents=True) + (dist_info / "METADATA").write_bytes(metadata) + + found = detect_framework(folder, package_dir=tmp_path / "pkg") + + assert found["version"] == "1.0" + + +def test_version_read_is_null_for_a_result_with_no_row(tmp_path: Path) -> None: + folder = tmp_path / "agent" + _write(folder, {"main.py": "from crewai import Crew\n"}) + package_dir = tmp_path / "package" + _dist(package_dir, "crewai-1.0", _metadata("1.0")) + + assert detect_framework(folder, package_dir=package_dir) == UNKNOWN + + +@pytest.mark.parametrize( + "metadata", + [ + b"Metadata-Version: 2.1\nVersion: 1.0\x00\n", + b"Metadata-Version: 2.1\nVersion: sk_live_x\n", + b"Metadata-Version: 2.1\nVersion: 1" + b".0" * 32 + b"\n", + b"Metadata-Version: 2.1\nName: crewai\n", + b"Metadata-Version: 2.1\nX-Version: 1.0\nNotVersion: 2.0\n", + b"Metadata-Version: 2.1\nVersion: caf\xc3\xa9\n", + b"Metadata-Version: 2.1\nVersion: bad\nVersion: 1.0\n", + ], + ids=[ + "nul", + "secret_like", + "too_long", + "no_version_line", + "not_at_line_start", + "non_ascii", + "later_line_not_read", + ], +) +def test_bad_metadata_gives_null_version(tmp_path: Path, metadata: bytes) -> None: + folder = tmp_path / "agent" + _write(folder, {"main.py": CREW}) + package_dir = tmp_path / "package" + _dist(package_dir, "crewai-1.0", metadata) + + assert detect_framework(folder, package_dir=package_dir) == _named("crewai") + + +def test_bad_metadata_gives_null_version_symlink(tmp_path: Path) -> None: + folder = tmp_path / "agent" + _write(folder, {"main.py": CREW}) + package_dir = tmp_path / "package" + elsewhere = tmp_path / "METADATA" + elsewhere.write_bytes(_metadata("1.0")) + (_dist(package_dir, "crewai-1.0") / "METADATA").symlink_to(elsewhere) + + assert detect_framework(folder, package_dir=package_dir) == _named("crewai") + + +def test_metadata_read_stops_at_64_kib(tmp_path: Path) -> None: + folder = tmp_path / "agent" + _write(folder, {"main.py": CREW}) + package_dir = tmp_path / "package" + padding = b"Metadata-Version: 2.1\n" + b"X: y\n" * 14_000 + assert len(padding) > fd._METADATA_BYTES + _dist(package_dir, "crewai-1.0", padding + b"Version: 1.0\n") + + assert detect_framework(folder, package_dir=package_dir) == _named("crewai") + + +def _process_manifest() -> SimpleNamespace: + return SimpleNamespace( + runtime=SimpleNamespace(kind="process", control_service="agent"), + processes=[ + SimpleNamespace(name="postgres", working_directory=None), + SimpleNamespace(name="agent", working_directory="generated/source/agent"), + ], + ) + + +def _compose_manifest() -> SimpleNamespace: + return SimpleNamespace( + runtime=SimpleNamespace(kind="compose", control_service=None), processes=[] + ) + + +@pytest.mark.parametrize( + ("subfolder", "contract", "manifest", "expected"), + [ + ("sub", {"runtime": {"workdir": "inner"}}, _process_manifest(), "source/sub"), + (None, {"runtime": {"workdir": "inner"}}, _process_manifest(), "source/inner"), + (None, {"runtime": {"workdir": "inner"}}, _compose_manifest(), "source/inner"), + ( + None, + {"runtime": {"workdir": "."}}, + _process_manifest(), + "bundle/generated/source/agent", + ), + ( + None, + {"runtime": {"workdir": ""}}, + _process_manifest(), + "bundle/generated/source/agent", + ), + ( + None, + {"runtime": {"workdir": 3}}, + _process_manifest(), + "bundle/generated/source/agent", + ), + ( + None, + {"runtime": {"workdir": "../outside"}}, + _process_manifest(), + "bundle/generated/source/agent", + ), + ( + None, + {"runtime": {"workdir": "a_file"}}, + _process_manifest(), + "bundle/generated/source/agent", + ), + (None, None, _process_manifest(), "bundle/generated/source/agent"), + (None, "{not json", _process_manifest(), "bundle/generated/source/agent"), + (None, ["a", "list"], _process_manifest(), "bundle/generated/source/agent"), + (None, None, _compose_manifest(), "source"), + ], + ids=[ + "subfolder_first", + "workdir_before_process", + "workdir_on_compose", + "dot_workdir", + "empty_workdir", + "non_string_workdir", + "workdir_escaping_source", + "workdir_is_a_file", + "missing_contract", + "corrupt_contract", + "contract_not_an_object", + "compose_falls_to_source", + ], +) +def test_folder_order( + tmp_path: Path, + subfolder: str | None, + contract: Any, + manifest: SimpleNamespace, + expected: str, +) -> None: + source = tmp_path / "source" + bundle_dir = tmp_path / "bundle" + work = tmp_path / "work" + _write( + tmp_path, + { + "source/a_file": "", + "source/inner/x.py": "", + "source/sub/x.py": "", + "outside/x.py": "", + }, + ) + bundle_dir.mkdir() + if isinstance(contract, str): + (bundle_dir / "contract.json").write_text(contract, encoding="utf-8") + elif contract is not None: + (bundle_dir / "contract.json").write_text( + json.dumps(contract), encoding="utf-8" + ) + job = SimpleNamespace(source=SimpleNamespace(subfolder=subfolder)) + + folders = framework_folders_for( + job, manifest, source=source, bundle_dir=bundle_dir, work_directory=work + ) + + is_process = manifest.runtime.kind == "process" + assert folders == FrameworkFolders( + tmp_path / expected, source, work / "build" / "agent" if is_process else None + ) + + +def _github_job(subfolder: str | None) -> SimpleNamespace: + return SimpleNamespace( + source=RepositorySource( + kind=SourceKind.GITHUB, + repository="org/repo", + visibility=SourceVisibility.PUBLIC, + commit_sha="a" * 40, + subfolder=subfolder, + ) + ) + + +@pytest.mark.parametrize( + ("subfolder", "expected"), + [(None, "source"), ("inner", "source/inner")], + ids=["no_subfolder", "subfolder"], +) +def test_folder_order_on_a_real_job_source( + tmp_path: Path, subfolder: str | None, expected: str +) -> None: + source = tmp_path / "source" + _write(tmp_path, {"source/inner/x.py": ""}) + + folders = framework_folders_for( + _github_job(subfolder), + _compose_manifest(), + source=source, + bundle_dir=tmp_path / "bundle", + work_directory=tmp_path / "work", + ) + + assert folders == FrameworkFolders(tmp_path / expected, source, None) + + +def test_folder_order_refuses_a_subfolder_outside_the_source(tmp_path: Path) -> None: + _write(tmp_path, {"source/x.py": "", "outside/x.py": ""}) + + with pytest.raises(SubfolderNotFound): + framework_folders_for( + _github_job("../outside"), + _compose_manifest(), + source=tmp_path / "source", + bundle_dir=tmp_path / "bundle", + work_directory=tmp_path / "work", + ) + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + ("import livekit\nlivekit.agents.AgentSession()\n", _named("livekit")), + ("import google\ngoogle.adk.agents.Agent()\n", _named("google_adk")), + ("import google\ngoogle.genai.Client()\n", NONE), + ("import livekit\nlivekit.agents.JobContext()\n", UNKNOWN), + ("import livekit\nagents = livekit.agents\n", UNKNOWN), + ("import livekit\nprint(livekit.agents.__version__)\n", UNKNOWN), + ("import google\ngoogle.adk.tools.FunctionTool(f)\n", UNKNOWN), + ("import google\nadk = google.adk\n", UNKNOWN), + ("from livekit import api\napi.AccessToken()\n", NONE), + ], + ids=[ + "livekit", + "google_adk", + "google_not_adk", + "livekit_no_builder", + "livekit_assigned", + "livekit_read_only", + "google_adk_no_builder", + "google_adk_assigned", + "livekit_not_the_row", + ], +) +def test_top_name_import_then_dotted_use( + tmp_path: Path, source: str, expected: dict[str, Any] +) -> None: + _write(tmp_path, {"main.py": source}) + + assert detect_framework(tmp_path) == expected diff --git a/tests/harness/test_hosted_entrypoint.py b/tests/harness/test_hosted_entrypoint.py index 991bcc5e1..76140cc39 100644 --- a/tests/harness/test_hosted_entrypoint.py +++ b/tests/harness/test_hosted_entrypoint.py @@ -5089,3 +5089,121 @@ def test_bad_subfolder_fails_terminal_before_compose(monkeypatch): "code": "subfolder_not_found", "message": subfolder_not_found("agents/missing").message, } + + +def _harness_with_a_packaged_agent( + packaged_main: bytes = b"from crewai import Crew\nCrew()\n", +) -> Harness: + real_body = _base_manifest_body + + def packaged_body() -> dict[str, Any]: + body = real_body() + body["processes"][1]["working_directory"] = "generated/source/agent" + return body + + files = { + **_scenario_doc_files( + "passing", + scenario_key="passing", + scenario_id="platform-passing", + sub_goals=["holds"], + checks={"holds": "def check(world, calls):\n return None\n"}, + ), + "contract.json": json.dumps({"agent": "crew-bot", "tools": []}).encode("utf-8"), + "generated/source/agent/main.py": packaged_main, + } + with mock.patch.dict(globals(), {"_base_manifest_body": packaged_body}): + return _build_harness( + scenarios=[], + instances=1, + use_default_scenario_source=True, + bundle_writer=lambda bundle_dir: _write_bundle_with_scenario_files( + bundle_dir, files + ), + ) + + +def _posted_framework(harness: Harness) -> dict[str, Any]: + [document] = [ + body["agent"]["agent"] + for _, body in harness.transport.scenarios_calls + if "agent" in body + ] + return document["extension"]["framework"] + + +def _install_crewai(package_dir: Path) -> None: + dist_info = ( + package_dir + / ".venv" + / "lib" + / "python3.12" + / "site-packages" + / "crewai-0.86.0.dist-info" + ) + dist_info.mkdir(parents=True) + (dist_info / "METADATA").write_bytes( + b"Metadata-Version: 2.1\nName: crewai\nVersion: 0.86.0\n" + ) + + +def _framework_log_lines(caplog: pytest.LogCaptureFixture) -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if record.getMessage().startswith("framework") + ] + + +def test_run_job_gives_bundle_source_its_folders(caplog) -> None: + async def scenario() -> None: + harness = _harness_with_a_packaged_agent() + _install_crewai(harness.work / "build" / "agent") + + with mock.patch.object( + he, "framework_folders_for", wraps=he.framework_folders_for + ) as folders_for: + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + + assert code == he.EXIT_OK + assert _posted_framework(harness) == {"name": "crewai", "version": "0.86.0"} + assert folders_for.call_args.kwargs["source"] == harness.source + lines = _framework_log_lines(caplog) + assert any("name=crewai" in line for line in lines) + assert not any("0.86.0" in line for line in lines) + + with caplog.at_level("INFO", logger="fi.alk.harness"): + asyncio.run(scenario()) + + +def test_folder_step_failure_keeps_the_run_going(caplog) -> None: + secret = "/customer/private/path" + + async def scenario() -> None: + # The crew only in `source`, and a version where each wrong fallback would read + # one: the fallback is the source, with no build tree. + harness = _harness_with_a_packaged_agent(packaged_main=b"print('plain')\n") + (harness.source / "main.py").write_text( + "from crewai import Crew\nCrew()\n", encoding="utf-8" + ) + _install_crewai(harness.work / "build" / "agent") + _install_crewai(harness.work) + + with mock.patch.object( + he, "framework_folders_for", side_effect=RuntimeError(secret) + ): + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + + assert code == he.EXIT_OK + assert _posted_framework(harness) == {"name": "crewai", "version": None} + assert len(harness.transport.terminal_events()) == 1 + lines = _framework_log_lines(caplog) + assert any("class=RuntimeError" in line for line in lines) + assert not any(secret in line for line in lines) + + with caplog.at_level("INFO", logger="fi.alk.harness"): + asyncio.run(scenario()) diff --git a/tests/harness/test_scenario_source_provision_agent.py b/tests/harness/test_scenario_source_provision_agent.py index e11bc6030..8f72762f8 100644 --- a/tests/harness/test_scenario_source_provision_agent.py +++ b/tests/harness/test_scenario_source_provision_agent.py @@ -12,6 +12,7 @@ import fi.alk from fi.alk.harness import scenario_source as ss from fi.alk.harness.contract import AgentContract +from fi.alk.harness.framework_detection import FrameworkFolders from fi.alk.harness.job import SourceKind from fi.alk.providers.agent import AgentBuildError, build_authored_agent @@ -57,7 +58,11 @@ def _contract(prompt: str) -> dict[str, Any]: def _provision( - bundle_dir: Path, kind: SourceKind, subfolder: str | None = None + bundle_dir: Path, + kind: SourceKind, + source: ss.BundleScenarioSource | None = None, + *, + subfolder: str | None = None, ) -> dict[str, Any]: client = _RecordingClient() job = SimpleNamespace( @@ -66,7 +71,7 @@ def _provision( source=SimpleNamespace(kind=kind, repository="org/clinic", subfolder=subfolder), ) asyncio.run( - ss.BundleScenarioSource().build( + (source or ss.BundleScenarioSource()).build( job, object(), client, @@ -233,3 +238,129 @@ def test_no_subfolder_setup_hash_is_pinned(tmp_path: Path) -> None: payload["agent"]["agent"]["setup_hash"] == "sh1:27408adef9319e5f760d442d8e1e4c34ca2787f736e49ede86bc7398b323dd7b" ) + + +def _source_with_crew(tmp_path: Path) -> ss.BundleScenarioSource: + agent = tmp_path / "agent" + agent.mkdir() + (agent / "main.py").write_text( + "from crewai import Crew\nCrew()\n", encoding="utf-8" + ) + return ss.BundleScenarioSource( + framework_folders=FrameworkFolders(agent, agent, None) + ) + + +def _framework_log_lines(caplog: pytest.LogCaptureFixture) -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if record.getMessage().startswith("framework") + ] + + +def test_agent_document_carries_framework( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + bundle = tmp_path / "bundle" + contract = _contract("p") + _write_bundle(bundle, contract) + + with caplog.at_level(logging.INFO, logger="fi.alk.harness"): + payload = _provision(bundle, SourceKind.GITHUB, _source_with_crew(tmp_path)) + + lines = _framework_log_lines(caplog) + assert any("name=crewai" in line for line in lines) + assert not any(str(tmp_path) in line or "main.py" in line for line in lines) + framework = {"name": "crewai", "version": None} + assert payload["agent"]["agent"]["extension"]["framework"] == framework + assert payload["agent"]["agent"] == build_authored_agent( + contract, framework=framework + ) + + +def test_detector_crash_keeps_document( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + bundle = tmp_path / "bundle" + _write_bundle(bundle, _contract("p")) + secret = "/customer/private/path" + + def _raise(*_: Any, **__: Any) -> dict[str, Any]: + raise RuntimeError(secret) + + monkeypatch.setattr(ss, "detect_framework", _raise) + + with caplog.at_level(logging.INFO, logger="fi.alk.harness"): + payload = _provision(bundle, SourceKind.GITHUB, _source_with_crew(tmp_path)) + + lines = _framework_log_lines(caplog) + assert any("class=RuntimeError" in line for line in lines) + assert not any(secret in line for line in lines) + assert payload["agent"]["agent"]["extension"]["framework"] == { + "name": "unknown", + "version": None, + } + + +def test_no_folders_omits_framework(tmp_path: Path) -> None: + _write_bundle(tmp_path, _contract("p")) + + payload = _provision(tmp_path, SourceKind.GITHUB) + + assert set(payload["agent"]["agent"]["extension"]) == {"contract"} + + +@pytest.mark.parametrize( + ("kind", "expected_calls"), + [(SourceKind.GITHUB, 1), (SourceKind.ARCHIVE, 1), (SourceKind.REMOTE, 0)], + ids=["github", "archive", "not_for_a_remote_source"], +) +def test_detection_runs_once_off_the_event_loop( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + kind: SourceKind, + expected_calls: int, +) -> None: + bundle = tmp_path / "bundle" + _write_bundle(bundle, _contract("p")) + detections: list[Any] = [] + real_to_thread = asyncio.to_thread + + async def spy(function: Any, /, *args: Any, **kwargs: Any) -> Any: + if function is ss.detect_framework: + detections.append(args) + return await real_to_thread(function, *args, **kwargs) + + monkeypatch.setattr(ss.asyncio, "to_thread", spy) + + _provision(bundle, kind, _source_with_crew(tmp_path)) + + assert len(detections) == expected_calls + + +def test_detection_gets_all_three_folders( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + bundle = tmp_path / "bundle" + _write_bundle(bundle, _contract("p")) + calls: list[tuple[Any, dict[str, Any]]] = [] + + def record(folder: Any, **kwargs: Any) -> dict[str, Any]: + calls.append((folder, kwargs)) + return {"name": "none", "version": None} + + monkeypatch.setattr(ss, "detect_framework", record) + folders = FrameworkFolders(tmp_path / "a", tmp_path / "b", tmp_path / "c") + + _provision( + bundle, + SourceKind.GITHUB, + ss.BundleScenarioSource(framework_folders=folders), + ) + + assert calls == [ + (tmp_path / "a", {"repo_root": tmp_path / "b", "package_dir": tmp_path / "c"}) + ] diff --git a/tests/providers/test_providers_agent_authored.py b/tests/providers/test_providers_agent_authored.py index 677e24b20..312b9f9ff 100644 --- a/tests/providers/test_providers_agent_authored.py +++ b/tests/providers/test_providers_agent_authored.py @@ -166,3 +166,24 @@ def test_authored_keeps_the_host_of_a_url_with_a_placeholder_port() -> None: assert stored["braces"] == "http://{host}/v1" assert stored["shell"] == "http://localhost/v1" assert stored["numeric"] == "http://localhost:8080/v1" + + +def test_framework_written_beside_contract() -> None: + contract = _contract() + framework = {"name": "other", "version": "1.2", "label": "dspy"} + + agent = build_authored_agent(contract, framework=framework) + + assert set(agent["extension"]) == {"contract", "framework"} + assert agent["extension"]["framework"] == framework + assert agent["setup_hash"] == compute_setup_hash(agent) + assert agent["setup_hash"] != build_authored_agent(contract)["setup_hash"] + + +def test_no_framework_leaves_extension_and_hash() -> None: + recorded = json.loads((AUTHORED / "chat_source.json").read_text()) + + agent = build_authored_agent(recorded["contract"], framework=None) + + assert agent == recorded["agent"] + assert set(agent["extension"]) == {"contract"}