diff --git a/.github/workflows/sdk-smoke.yml b/.github/workflows/sdk-smoke.yml index 93f3c35a6..4c1f62f78 100644 --- a/.github/workflows/sdk-smoke.yml +++ b/.github/workflows/sdk-smoke.yml @@ -105,6 +105,14 @@ jobs: tests/harness/test_instrumentation_evidence.py tests/harness/test_agent_type_refusal.py tests/harness/test_executor_completed_counts.py + tests/harness/test_reading_entry.py + tests/harness/test_agent_reading.py + tests/harness/test_scan_walk.py + tests/harness/test_scan_folders.py + tests/harness/test_scan_document.py + tests/harness/test_scan_ai.py + tests/harness/test_agent_scan.py + tests/harness/test_authoring_entrypoint_find_agents.py tests/test_bump_deployment_values.py tests/test_release_workflow.py tests/test_release_pr_step.py diff --git a/src/fi/alk/harness/agent_reading.py b/src/fi/alk/harness/agent_reading.py new file mode 100644 index 000000000..7ce74e0b5 --- /dev/null +++ b/src/fi/alk/harness/agent_reading.py @@ -0,0 +1,362 @@ +"""Read one agent the way a build reads it, and leave its description, with no build after it. + +The platform shows an agent's description before any environment exists, and bills the reading +from ``cost.json``. So the reading runs the build's own stage and gate, and every way it can end +leaves the files the platform judges it by: a saved contract is never thrown away and paid for +again, a timeout is never retried as a crash, and the cost on disk is never marked complete +while some of the spend may be missing from it. +""" + +from __future__ import annotations + +import asyncio +import functools +import logging +import time +from dataclasses import replace +from pathlib import Path +from typing import Any + +from . import session +from .agent_type_check import EXIT_REFUSED +from .backends import StageDone +from .cli import guidance_text +from .failure_file import FAILURE_FILE +from .job import FailureDomain, HarnessFailure, HarnessJob, ProviderExecutionMode, SourceKind +from .reading_entry import ( + EXIT_RETRYABLE, + READING_JOB_INVALID, + MaskSecrets, + ReadingMode, + limit_seconds, + reading_failure, + write_reading_cost, + write_reading_failure, +) +from .reading_result import ( + CONTRACT_FILE, + classify, + document, + model_error, + out_of_turns, + saved_file, +) +from .session import Stage, StageIdleTimeout +from .source_folder import refuse_unusable_folder +from .sources import resolve as resolve_source +from .understand import SUBMIT_REMINDER, open_stage, opening, reading_settings + +logger = logging.getLogger(__name__) +logger.addFilter(MaskSecrets()) + +READING_SOURCE_EMPTY = "reading_source_empty" +AGENT_READING_TIMED_OUT = "agent_reading_timed_out" +AGENT_READING_CRASHED = "agent_reading_crashed" + +_LIMIT_ENV = "ALK_READING_MAX_DURATION_SECONDS" +_DEFAULT_SOFT_LIMIT_SECONDS = 1800.0 +# The hard limit outlasts the soft one by the longest stop (the interrupt, the wait after it and +# the close: 90 s) and room for the saved contract's document after it, whose framework scan can +# take well over a minute on a crafted repository, so a reading that stops itself is never cut +# off by the helper first. +_HARD_LIMIT_MARGIN_SECONDS = 300.0 +_INTERRUPT_SECONDS = 10.0 +_INTERRUPT_WAIT_SECONDS = 60.0 +# Ours, not the SDK's: its close is shielded and has no bound of its own. With it the decision +# comes by soft + 90 s, which leaves the rest of the margin to the document. +_CLOSE_SECONDS = 20.0 + +_COUNTS = ("tokens_in", "tokens_out", "tokens_cached") + + +def _soft_limit() -> float: + # The helper reads the same variable the same way for its own limit. + return limit_seconds(_LIMIT_ENV, _DEFAULT_SOFT_LIMIT_SECONDS) + + +def _check_job(job: HarnessJob) -> HarnessFailure | None: + if job.agent.mode in (ProviderExecutionMode.PROVIDER_IMPORT, ProviderExecutionMode.CONNECT_ONLY): + # The build reads these agents from the provider's definition, which a reading never loads. + return reading_failure( + FailureDomain.PLATFORM_SYNC, + READING_JOB_INVALID, + "A reading without a build reads an agent from its repository only. A " + f"{job.agent.mode.value} job's agent is defined by its provider, so it needs a build.", + retryable=False, + ) + if job.source.kind in (SourceKind.GITHUB, SourceKind.ARCHIVE): + return None + return reading_failure( + FailureDomain.PLATFORM_SYNC, + READING_JOB_INVALID, + f"An agent is read from a GitHub repository or an upload, not from a " + f"{job.source.kind.value} source.", + retryable=False, + ) + + +class _ReadingStage(Stage): + """The build's reading stage, with each turn's own spend kept, and no turn ever replayed.""" + + def __init__( + self, + spec: Any, + *, + name: str, + backend: Any, + running_totals: bool, + ) -> None: + # Never replayed after a quiet stream: a replayed turn is paid for again, and a new + # session would restart Claude's running totals under the baseline the difference is + # taken from. + super().__init__(spec, name=name, backend=backend, idle_retries=0) + self.running_totals = running_totals + self.done: list[StageDone] = [] + self.in_flight = False + self.opened = False + self._last_cost = 0.0 + self._last_counts = dict.fromkeys(_COUNTS, 0) + + async def __aenter__(self) -> "_ReadingStage": + await super().__aenter__() + self.opened = True + return self + + def _own(self, reported: StageDone) -> StageDone: + # Claude reports the session's running total on every result, so adding results as + # reported would count every earlier turn again. + if not self.running_totals: + return reported + cost = reported.cost_usd + if cost is not None: + cost, self._last_cost = max(0.0, cost - self._last_cost), cost + counts: dict[str, int] = {} + for key in _COUNTS: + total = int(getattr(reported, key) or 0) + counts[key] = max(0, total - self._last_counts[key]) + self._last_counts[key] = total + return replace(reported, cost_usd=cost, **counts) + + def _events(self, received: Any, turn: Any) -> list[Any]: + if not isinstance(received, StageDone): + return super()._events(received, turn) + done = self._own(received) + events = super()._events(done, turn) + self.done.append(done) + self.in_flight = False + return events + + async def say(self, message: str, **kwargs: Any) -> Any: + self.in_flight = True + return await super().say(message, **kwargs) + + +def _source_has_files(source: Path) -> bool: + try: + return source.is_dir() and next(source.iterdir(), None) is not None + except OSError: + # Read as empty, so retryable: a source the sandbox can't list yet may be listable next time. + return False + + +def _reminder_due(stage: _ReadingStage, output: Path, stopping: asyncio.Event) -> bool: + if saved_file(output) or stopping.is_set(): + return False + opened = stage.done[-1] if stage.done else None + # An error flag with no status may have passed; one nudge costs less than a whole retry. + return opened is None or ( + opened.api_error_status is None and not out_of_turns(opened) + ) + + +async def _converse( + stage: _ReadingStage, message: str, output: Path, stopping: asyncio.Event +) -> None: + # The session lives inside this task, so cancelling the task closes it. + async with stage: + await stage.say(message) + if _reminder_due(stage, output, stopping): + await stage.say(SUBMIT_REMINDER) + + +async def _stop( + stage: _ReadingStage, task: asyncio.Task[None], stopping: asyncio.Event, wait: float +) -> None: + # Set first, so a turn ending now doesn't earn the reminder. + stopping.set() + # Its own task, waited on with a bound: wait_for would wait again for a cancel the interrupt + # swallows. A Ctrl-C here reaches `_read`, which lets it through. + interrupting = asyncio.ensure_future(stage.interrupt_response()) + bound = min(wait, _INTERRUPT_SECONDS) + await asyncio.wait({interrupting}, timeout=bound) + if not interrupting.done(): + interrupting.cancel() + logger.warning("the stop at the time limit was not delivered within %g s", bound) + elif interrupting.cancelled() or interrupting.exception() is not None: + # A stop that can't be delivered is still a timeout, not a crash. + logger.warning( + "the stop at the time limit was not delivered", + exc_info=None if interrupting.cancelled() else interrupting.exception(), + ) + await asyncio.wait({task}, timeout=wait) + # Closed before the files are read, so a late tool call is unlikely to save after the + # decision. The close is bounded, though: a session that hangs closing must not keep the + # reading from deciding, and one that outlives the bound could still act afterwards. + task.cancel() + closing = min(wait, _CLOSE_SECONDS) + await asyncio.wait({task}, timeout=closing) + if not task.done(): + logger.warning("the stopped reading's session did not close within %g s", closing) + elif not task.cancelled() and task.exception() is not None: + logger.warning("the stopped reading ended with an error", exc_info=task.exception()) + + +def _timed_out(output: Path, soft_limit: float) -> int: + write_reading_failure( + reading_failure( + FailureDomain.INFRASTRUCTURE, + AGENT_READING_TIMED_OUT, + f"The agent reading ran past its {soft_limit:g} s limit without saving the agent's " + "description, and was stopped.", + retryable=False, + ), + output, + ) + return EXIT_REFUSED + + +async def _reading( + job: HarnessJob, source: Path, output: Path, started: float, soft_limit: float, wait: float +) -> tuple[int, bool]: + """The exit code, and whether every turn's spend is known to be in the cost file.""" + # Before the folder check, so an upload that hasn't arrived isn't blamed on the folder. + if not _source_has_files(source): + write_reading_failure( + reading_failure( + FailureDomain.PLATFORM_SYNC, + READING_SOURCE_EMPTY, + "The agent's source was missing or empty, so there was nothing to read. " + "It may not have arrived yet.", + retryable=True, + ), + output, + ) + return EXIT_RETRYABLE, True + if refuse_unusable_folder(source, job.source.subfolder, output): + return EXIT_REFUSED, True + + backend = session.resolve() + agent_source = resolve_source( + "repo", + name=str(job.metadata.get("agent_name") or source.name), + root=source, + subfolder=job.source.subfolder, + ) + # Opened as the build opens its own reading, turns, thinking and setup rule included, so a + # contract saved here is one the build's gate would have kept. + opened, _ = open_stage(agent_source, out=output, **reading_settings(job)) + stage = _ReadingStage( + opened.spec, + name=opened.name, + backend=backend, + running_totals=getattr(backend, "name", "") == "claude", + ) + stopping = asyncio.Event() + # The build's opening, the job's guidance included: a generic job's runtime advice changes + # what the contract records. + message = opening(agent_source) + guidance_text(job.authoring_guidance()) + task = asyncio.create_task(_converse(stage, message, output, stopping)) + finished, _ = await asyncio.wait( + {task}, timeout=max(0.0, started + soft_limit - time.monotonic()) + ) + + if not finished: + await _stop(stage, task, stopping, wait) + # A request cut off by the stop may be billed without being in the last total. + saved = saved_file(output) + if saved == FAILURE_FILE: + return EXIT_REFUSED, False + if saved == CONTRACT_FILE: + return document(job, source, output), False + return _timed_out(output, soft_limit), False + + error = task.exception() + if error is not None and not stage.opened: + # Nothing reached the provider, so the fault is the reading's own: a crash, retried once. + raise error + if isinstance(error, StageIdleTimeout): + # The quiet turn's spend never arrived; what was saved before it still stands. + saved = saved_file(output) + if saved == FAILURE_FILE: + return EXIT_REFUSED, False + if saved == CONTRACT_FILE: + return document(job, source, output), False + return ( + model_error( + output, + "The model stopped responding before the agent's description was saved " + f"({error}).", + retryable=True, + ), + False, + ) + # A request cut with no status may still be billed, and its total never reached us. + complete = ( + error is None + and not stage.in_flight + and not any( + done.is_error and done.api_error_status is None and not out_of_turns(done) + for done in stage.done + ) + ) + return classify(job, source, output, stage, error), complete + + +async def _read(job: HarnessJob, source: Path, output: Path, *, interrupt_wait: float) -> int: + # The same instant the helper's hard limit counts from, give or take a few lines. + started = time.monotonic() + output = Path(output) + code, complete = EXIT_RETRYABLE, False + try: + try: + code, complete = await _reading( + job, Path(source), output, started, _soft_limit(), interrupt_wait + ) + except (Exception, asyncio.CancelledError) as exc: + # A contract the gate saved is paid for; a crash would have the platform pay again. + # A cancel leaking out of the session is the same: the saved contract still stands. + # A cancel of this task itself (a Ctrl-C) is let through, the saved files as they are. + # (3.10 has no `cancelling`, and there a Ctrl-C is raised as itself, not as a cancel.) + current = asyncio.current_task() + if saved_file(output) != CONTRACT_FILE or ( + isinstance(exc, asyncio.CancelledError) + and getattr(current, "cancelling", lambda: 0)() + ): + raise + logger.warning( + "reading fault after a saved contract: class=%s", type(exc).__name__, exc_info=True + ) + code, complete = document(job, Path(source), output), False + finally: + # The work owns the final cost file, on every way out. + write_reading_cost(output, complete=complete) + return code + + +def _read_agent(*, interrupt_wait: float = _INTERRUPT_WAIT_SECONDS) -> ReadingMode: + return ReadingMode( + name="read_agent", + check_job=_check_job, + work=functools.partial(_read, interrupt_wait=interrupt_wait), + # The helper reads the variable when the reading starts, as the work does, so its + # limit is always the reading's own plus the margin. + time_limit_seconds=_DEFAULT_SOFT_LIMIT_SECONDS, + timed_out_code=AGENT_READING_TIMED_OUT, + timed_out_retryable=False, + crashed_code=AGENT_READING_CRASHED, + limit_env=_LIMIT_ENV, + limit_margin_seconds=_HARD_LIMIT_MARGIN_SECONDS, + ) + + +READ_AGENT = _read_agent() diff --git a/src/fi/alk/harness/agent_scan.py b/src/fi/alk/harness/agent_scan.py new file mode 100644 index 000000000..0446483dd --- /dev/null +++ b/src/fi/alk/harness/agent_scan.py @@ -0,0 +1,538 @@ +"""Finds the agents in a repository at one commit, or in an upload, and lists them for the platform. + +The platform shows this list before it creates an agent, and bills the scan from ``cost.json``. +A plain code check runs first and costs nothing. The model is asked only when that check finds +nothing outside test and example folders, or only files that call a model directly beside +TypeScript or JavaScript it can't read, because the customer pays for every turn of it. So +once the model has answered, nothing it was paid for is thrown away: a crash after it still +writes what was kept, and never ends in a retryable failure that would pay for the same answer +again. Every return leaves the final ``cost.json``, marked incomplete whenever the model's turn +may be missing from it. +""" + +from __future__ import annotations + +import logging +import os +import stat +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field, replace +from pathlib import Path + +from .job import FailureDomain, HarnessFailure, HarnessJob, SourceKind +from .agent_type_check import EXIT_REFUSED +from .reading_entry import ( + EXIT_DONE, + EXIT_RETRYABLE, + MaskSecrets, + ReadingMode, + limit_seconds, + reading_failure, + write_reading_cost, + write_reading_failure, +) +from .scan_ai import AiOutcome, run_ai_step +from .scan_document import Row, ScanResultInvalid, finish, root_name, write_agents_json +from .scan_folders import ClockRanOut, FolderChecker, FolderCheckerBroken +from .scan_walk import _WEAKEST, Candidate, Hazard, WalkResult, describe_entry, walk_repo + +logger = logging.getLogger(__name__) +logger.addFilter(MaskSecrets()) + +# Failures, written to failure.json. +SCAN_JOB_INVALID = "scan_job_invalid" +SCAN_SOURCE_MISSING = "scan_source_missing" +SCAN_CRASHED = "scan_crashed" +SCAN_HARD_LIMIT_REACHED = "scan_hard_limit_reached" +SCAN_MODEL_ERROR = "scan_model_error" +SCAN_MODEL_FORBIDDEN = "scan_model_forbidden" +SCAN_RESULT_INVALID = "scan_result_invalid" + +# Results, written to agents.json as its error. +SCAN_LIMIT_REACHED = "scan_limit_reached" +SCAN_TIMED_OUT = "scan_timed_out" +SCAN_OUT_OF_MEMORY = "scan_out_of_memory" +SCAN_AI_NO_ANSWER = "scan_ai_no_answer" +SCAN_FAILED_AFTER_AI = "scan_failed_after_ai" +AGENT_PATH_UNSUPPORTED = "agent_path_unsupported" + +CLOCK_ENV = "ALK_SCAN_MAX_SECONDS" +# The scan's clock when CLOCK_ENV doesn't name one. The helper reads the same variable for its +# hard stop, both when the reading starts. +CLOCK = 1200.0 +AI_ENV = "ALK_SCAN_AI_MAX_SECONDS" +# The model step's limit when AI_ENV doesn't name one; read when the model step is asked. +AI_SECONDS = 600.0 +# The helper's hard stop comes this long after the scan's own clock, so a scan that stops itself +# always writes its own file first. +HARD_LIMIT_MARGIN = 600.0 +# The model is asked only with this much left: its own time, then what follows it. +AI_LEAST_LEFT = 360.0 +# Kept back from the model for the stop sequence, the AI rows' descriptions and their checks. +AI_RESERVE = 180.0 + +_ERROR_MESSAGES = { + SCAN_LIMIT_REACHED: "The repository is larger than the scan reads, and no agent was found " + "before the limit.", + SCAN_TIMED_OUT: "The scan ran out of time before it found an agent.", + SCAN_OUT_OF_MEMORY: "The scan ran out of memory before it found an agent.", + SCAN_AI_NO_ANSWER: "The model looked for agents and gave no answer.", + SCAN_FAILED_AFTER_AI: "The scan failed after the model looked for agents, and nothing it " + "found could be kept.", + AGENT_PATH_UNSUPPORTED: "Every agent found sits at a path the platform cannot use.", +} +_STOPPED = {"limit": SCAN_LIMIT_REACHED, "clock": SCAN_TIMED_OUT, "memory": SCAN_OUT_OF_MEMORY} + +# A walk that ran out of memory is dropped whole, so nothing it decided is half kept. +_NO_WALK = WalkResult( + agents=(), + more_than_fifty=False, + stopped="memory", + unread_files=0, + source_outside_side=False, + candidate_outside_side=False, + script_outside_side=False, + only_weak_outside_side=False, + paths_left_out=0, + upload_others=0, + upload_others_side=0, +) + + +@dataclass +class _Scan: + """What one scan has so far, kept where the after-model catch can still reach it.""" + + upload: bool + walk_rows: list[Row] = field(default_factory=list) + ai_found: list[Candidate] = field(default_factory=list) + ai_rows: list[Row] = field(default_factory=list) + truncated: bool = False + cut_by_clock: bool = False + complete: bool = True + # One look per folder: many agents can share one, and the look walks skipped folders. + hazards: dict[str, Hazard] = field(default_factory=dict) + # The model was asked past walk rows that only call a model, so the cap must not drop + # what it found in their favour. + models_first: bool = False + + +def _fail( + output: Path, domain: FailureDomain, code: str, message: str, *, retryable: bool +) -> int: + write_reading_failure(reading_failure(domain, code, message, retryable=retryable), output) + logger.warning("agent scan: failed, code=%s retryable=%s", code, retryable) + return EXIT_RETRYABLE if retryable else EXIT_REFUSED + + +def _check_job(job: HarnessJob) -> HarnessFailure | None: + source = job.source + if source.kind in (SourceKind.REMOTE, SourceKind.PROVIDER) or source.subfolder: + return reading_failure( + FailureDomain.PLATFORM_SYNC, + SCAN_JOB_INVALID, + "The agent scan reads a whole GitHub repository or an upload; this job names a " + "remote or provider agent, or a folder.", + retryable=False, + ) + return None + + +def _holds_a_file(source: Path) -> bool: + """Whether ``source`` is a folder with at least one regular file, stopping at the first.""" + try: + if not source.is_dir(): + return False + except OSError: + return False + for directory, _, files in os.walk(source, followlinks=False): + for name in files: + try: + if stat.S_ISREG(os.lstat(os.path.join(directory, name)).st_mode): + return True + except OSError: + continue + return False + + +def _hints(walk: WalkResult) -> str: + """What the walk counted, for the model's opening. Counts only, never a path: a path invited + the model to submit again what the walk already listed.""" + side = sum(candidate.side for candidate in walk.agents) + walk.upload_others_side + # The model is asked past candidates outside side folders only when every one of them + # only calls a model, so each unlisted one outside them is one of those. + weak = sum( + not candidate.side and candidate.sign in _WEAKEST for candidate in walk.agents + ) + (walk.upload_others - walk.upload_others_side) + return ( + "source files outside those folders: yes; " + f"agent-like files inside them: {side}; " + f"files outside them that only call a model: {weak}; " + f"files the check could not read: {walk.unread_files}." + ) + + +def _unchecked(candidate: Candidate, *, found_by_ai: bool = True) -> Row: + return Row( + candidate=candidate, + framework={"name": "unknown", "version": None, "label": None}, + finished=False, + found_by_ai=found_by_ai, + ) + + +def _check_into( + rows: list[Row], + checker: FolderChecker, + source: Path, + candidates: Sequence[Candidate], + scan: _Scan, + *, + found_by_ai: bool, + deadline: float, + clock: Callable[[], float], +) -> None: + """Checks each candidate's folder in order, appending each row as soon as it is known.""" + for index, candidate in enumerate(candidates): + try: + word = checker.check(source, candidate.subfolder, deadline=deadline, clock=clock) + except ClockRanOut: + scan.truncated = scan.cut_by_clock = True + # Found, so listed without its folder's word rather than lost: the person can still + # pick it, and a build of it names the framework. + rows.extend( + _unchecked(left, found_by_ai=found_by_ai) for left in candidates[index:] + ) + return + rows.append(Row(candidate, word.framework, word.finished, found_by_ai)) + + +def _one_row(walk_rows: Sequence[Row], ai_rows: Sequence[Row]) -> tuple[list[Row], bool]: + """An upload's one row, at the root, and whether any other was left out.""" + every = [*walk_rows, *ai_rows] + if not every: + return [], False + outside = next((row for row in walk_rows if not row.candidate.side), None) + if outside is not None and outside.candidate.sign in _WEAKEST: + # The model was asked past it, so what the model found outside side folders is + # likelier the agent than a file that only calls a model. + outside = next((row for row in ai_rows if not row.candidate.side), outside) + chosen = ( + outside + or next((row for row in ai_rows if not row.candidate.side), None) + or every[0] + ) + if chosen.candidate.subfolder: + chosen = replace(chosen, candidate=replace(chosen.candidate, subfolder="")) + return [chosen], len(every) > 1 + + +def _write( + job: HarnessJob, + output: Path, + rows: Sequence[Row], + *, + upload: bool, + truncated: bool, + error: str | None, + models_first: bool, +) -> int: + try: + body, _ = finish( + rows=rows, + root_name=root_name(job), + commit_sha=None if upload else job.source.commit_sha, + truncated=truncated, + error=None if error is None else (error, _ERROR_MESSAGES[error]), + models_first=models_first, + ) + except ScanResultInvalid as invalid: + logger.error("agent scan: result invalid, problem=%s", invalid.problem) + return _fail( + output, + FailureDomain.INFRASTRUCTURE, + SCAN_RESULT_INVALID, + "The agent list broke the platform's rules and was not written; this is a fault " + "in the scan, not in the repository.", + retryable=False, + ) + write_agents_json(output, body) + logger.info( + "agent scan: written, rows=%d truncated=%s error=%s", + len(rows), + truncated, + error or "none", + ) + return EXIT_DONE + + +def _result( + job: HarnessJob, + output: Path, + scan: _Scan, + walk: WalkResult, + outcome: AiOutcome | None, +) -> int: + """Turn the rows and the model's outcome into the file: an upload keeps one row, the + outcome decides the status and ``truncated``, and the walk's flags add to ``truncated``.""" + if scan.upload: + rows, others = _one_row(scan.walk_rows, scan.ai_rows) + else: + rows, others = [*scan.walk_rows, *scan.ai_rows], False + truncated = scan.truncated or others + error: str | None = None + if outcome is not None: + if outcome.kind == "model_error": + logger.warning("agent scan: model step failed, %s", outcome.message) + if not rows: + return _fail( + output, + FailureDomain.INFRASTRUCTURE, + SCAN_MODEL_ERROR, + "The model failed while looking for agents; a retry may get through." + if outcome.retryable + else "The model refused the request to look for agents; a retry would be " + "refused the same way.", + retryable=outcome.retryable, + ) + # The walk's rows are already checked and cost nothing. A retry would pay for the + # model again to add at most what it missed, so the rows are kept, marked partial. + truncated = True + elif outcome.kind == "forbidden": + return _fail( + output, + FailureDomain.INFRASTRUCTURE, + SCAN_MODEL_FORBIDDEN, + "The provider billed a model this kit may not spend on, so the scan stopped.", + retryable=False, + ) + elif outcome.kind in ("no_answer", "timed_out"): + if rows: + truncated = True + else: + error = SCAN_AI_NO_ANSWER if outcome.kind == "no_answer" else SCAN_TIMED_OUT + truncated = truncated or outcome.refused > 0 or outcome.uncarried > 0 + truncated = ( + truncated + or walk.more_than_fifty + or bool(walk.stopped and rows) + or walk.paths_left_out > 0 + or walk.unread_files > 0 + or scan.cut_by_clock + or walk.upload_others > 0 + or any(row.candidate.partial or not row.finished for row in rows) + ) + if not rows and error is None: + if walk.stopped: + error = _STOPPED[walk.stopped] + elif walk.paths_left_out or (outcome is not None and outcome.uncarried): + error = AGENT_PATH_UNSUPPORTED + return _write( + job, + output, + rows, + upload=scan.upload, + truncated=truncated, + error=error, + models_first=scan.models_first, + ) + + +def _after_ai( + job: HarnessJob, + source: Path, + output: Path, + scan: _Scan, + walk: WalkResult, + outcome: AiOutcome, + checker: FolderChecker, + *, + deadline: float, + clock: Callable[[], float], +) -> int: + for folder, entry in outcome.pairs: + candidate = describe_entry( + source, + entry, + folder, + deadline=deadline, + upload=scan.upload, + clock=clock, + hazards=scan.hazards, + ) + if candidate is None: + scan.truncated = True + continue + scan.ai_found.append(candidate) + # Walk rows first, so a walk row always beats a model row for the same entry, and a + # duplicate is never checked twice. + taken = {row.candidate.entry for row in scan.walk_rows} + fresh: list[Candidate] = [] + for candidate in scan.ai_found: + if candidate.entry not in taken: + taken.add(candidate.entry) + fresh.append(candidate) + scan.ai_found = fresh + _check_into( + scan.ai_rows, + checker, + source, + fresh, + scan, + found_by_ai=True, + deadline=deadline, + clock=clock, + ) + return _result(job, output, scan, walk, outcome) + + +def _keep_what_was_paid_for( + job: HarnessJob, output: Path, scan: _Scan, crash: Exception +) -> int: + """After the model was paid for, a crash still ends in a file, never in a paid retry.""" + # Logged with its trace, as the reading helper logs a crash. Only the platform's secret + # values are masked; a repo path in the message is not. + logger.error("agent scan: failed after the model step", exc_info=crash) + checked = {row.candidate.entry for row in scan.ai_rows} + ai_rows = [ + *scan.ai_rows, + *(_unchecked(candidate) for candidate in scan.ai_found if candidate.entry not in checked), + ] + if scan.upload: + rows, _ = _one_row(scan.walk_rows, ai_rows) + else: + rows = [*scan.walk_rows, *ai_rows] + try: + body, _ = finish( + rows=rows, + root_name=root_name(job), + commit_sha=None if scan.upload else job.source.commit_sha, + truncated=True, + error=None + if rows + else (SCAN_FAILED_AFTER_AI, _ERROR_MESSAGES[SCAN_FAILED_AFTER_AI]), + models_first=scan.models_first, + ) + write_agents_json(output, body) + return EXIT_DONE + except Exception: + logger.error("agent scan: the kept rows could not be written", exc_info=True) + try: + body, _ = finish( + rows=[], + root_name=root_name(job), + commit_sha=None if scan.upload else job.source.commit_sha, + truncated=True, + error=(SCAN_FAILED_AFTER_AI, _ERROR_MESSAGES[SCAN_FAILED_AFTER_AI]), + ) + write_agents_json(output, body) + return EXIT_DONE + except Exception: + logger.error("agent scan: the fixed error could not be written", exc_info=True) + raise crash + + +async def _scan(job: HarnessJob, source: Path, output: Path, scan: _Scan) -> int: + clock = time.monotonic + # Taken first, so the source check counts against the clock too. + deadline = clock() + limit_seconds(CLOCK_ENV, CLOCK) + if not _holds_a_file(source): + return _fail( + output, + FailureDomain.PLATFORM_SYNC, + SCAN_SOURCE_MISSING, + "The source to scan is missing, is not a folder, or holds no file.", + retryable=True, + ) + try: + walk = walk_repo( + source, + deadline=deadline, + upload=scan.upload, + clock=clock, + hazards=scan.hazards, + ) + except MemoryError: + walk = _NO_WALK + logger.warning("agent scan: the walk ran out of memory and was dropped") + with FolderChecker() as checker: + try: + _check_into( + scan.walk_rows, + checker, + source, + walk.agents, + scan, + found_by_ai=False, + deadline=deadline, + clock=clock, + ) + except FolderCheckerBroken: + # No model has run yet, so a retry costs nothing that was paid for. + return _fail( + output, + FailureDomain.INFRASTRUCTURE, + SCAN_CRASHED, + "The folder check could not run, before any model was asked.", + retryable=True, + ) + # A Python file that only calls a model says nothing of an agent written in TypeScript + # or JavaScript beside it, which the walk can't list. + asks_the_model = ( + ( + not walk.candidate_outside_side + or (walk.only_weak_outside_side and walk.script_outside_side) + ) + and walk.source_outside_side + and walk.stopped == "" + and deadline - clock() >= AI_LEAST_LEFT + ) + if not asks_the_model: + return _result(job, output, scan, walk, None) + scan.models_first = walk.candidate_outside_side + outcome = await run_ai_step( + source, + hints=_hints(walk), + seconds=min(limit_seconds(AI_ENV, AI_SECONDS), deadline - clock() - AI_RESERVE), + on_turn_end=lambda: write_reading_cost(output, complete=False), + ) + if not outcome.complete: + scan.complete = False + # `_write` has already turned a ScanResultInvalid into its 78, so it never reaches here. + try: + return _after_ai( + job, + source, + output, + scan, + walk, + outcome, + checker, + deadline=deadline, + clock=clock, + ) + except Exception as crash: + return _keep_what_was_paid_for(job, output, scan, crash) + + +async def _work(job: HarnessJob, source: Path, output: Path) -> int: + scan = _Scan(upload=job.source.kind is SourceKind.ARCHIVE) + try: + return await _scan(job, Path(source), Path(output), scan) + finally: + # The one place the final cost is written, so no return can leave the helper's zero. + write_reading_cost(output, complete=scan.complete) + + +FIND_AGENTS = ReadingMode( + name="find_agents", + check_job=_check_job, + work=_work, + time_limit_seconds=CLOCK, + limit_env=CLOCK_ENV, + limit_margin_seconds=HARD_LIMIT_MARGIN, + timed_out_code=SCAN_HARD_LIMIT_REACHED, + timed_out_retryable=False, + crashed_code=SCAN_CRASHED, +) diff --git a/src/fi/alk/harness/authoring_entrypoint.py b/src/fi/alk/harness/authoring_entrypoint.py index e019b686f..20d68a9c0 100644 --- a/src/fi/alk/harness/authoring_entrypoint.py +++ b/src/fi/alk/harness/authoring_entrypoint.py @@ -679,11 +679,50 @@ def main(argv: list[str] | None = None, *, validate_runtime: bool = False) -> in type=Path, help="Directory the sealed bundle is written to in --certify-only mode", ) + reading = parser.add_mutually_exclusive_group() + reading.add_argument( + "--read-agent-only", + action="store_true", + help="Read the agent and leave its description in --output; build nothing", + ) + reading.add_argument( + "--find-agents", + action="store_true", + help="Find the agents in the source and leave their list in --output; build nothing", + ) args = parser.parse_args(argv) if args.certify_only: if args.bundle_output is None: parser.error("--certify-only requires --bundle-output") return _certify_only(args) + if args.read_agent_only: + import logging + + from .agent_reading import READ_AGENT + from .reading_entry import run_reading_mode + + # The reading's start line is how its start-up is timed, and nothing else in this process + # sets up logging, so its INFO lines would otherwise be dropped. + reading_log = logging.getLogger("fi.alk.harness.reading_entry") + reading_log.setLevel(logging.INFO) + if not logging.getLogger().handlers: + reading_log.addHandler(logging.StreamHandler()) + return run_reading_mode( + READ_AGENT, + job_path=args.job, + source=args.source.resolve(), + output=args.output.resolve(), + ) + if args.find_agents: + from .agent_scan import FIND_AGENTS + from .reading_entry import run_reading_mode + + return run_reading_mode( + FIND_AGENTS, + job_path=args.job, + source=args.source.resolve(), + output=args.output.resolve(), + ) job = HarnessJob.model_validate(json.loads(args.job.read_text(encoding="utf-8"))) if refuse_unusable_folder( diff --git a/src/fi/alk/harness/cli.py b/src/fi/alk/harness/cli.py index 9bfad9a80..60971fc5f 100644 --- a/src/fi/alk/harness/cli.py +++ b/src/fi/alk/harness/cli.py @@ -15,6 +15,7 @@ import sys import time import uuid +from collections.abc import Iterable from pathlib import Path from typing import Any @@ -39,7 +40,7 @@ from .session import TEXT, Event from .sessions import Session, new_id, save as save_session from .sources import resolve, supported -from .understand import load, open_stage, opening +from .understand import SUBMIT_REMINDER, load, open_stage, opening, reading_settings from .world.snapshot import saved as world_saved @@ -168,11 +169,12 @@ async def _ask_operator(_tool_name: str, payload: dict[str, Any], _context: Any) def _guidance(args: argparse.Namespace) -> str: - instructions = [ - str(item).strip() - for item in (getattr(args, "guidance", None) or []) - if str(item).strip() - ] + return guidance_text(getattr(args, "guidance", None) or []) + + +def guidance_text(guidance: Iterable[Any]) -> str: + """A job's guidance as every stage that reads it is told it; empty when there is none.""" + instructions = [str(item).strip() for item in guidance if str(item).strip()] if not instructions: return "" return ( @@ -287,23 +289,13 @@ async def _understand(args: argparse.Namespace) -> int: ) else: source = resolve(args.kind, name=args.name, root=args.path, subfolder=subfolder) - offered = ( - (getattr(job, "metadata", None) or {}).get("available_evals") if job else None - ) - generic = bool( - job is not None - and isinstance(getattr(job, "metadata", None), dict) - and job.metadata.get("generic_harness_v1") is True - ) stage, destination = open_stage( source, out=Path(args.out) if args.out else None, # Unattended, there is nobody to answer, so the model records what it could not # resolve in open_questions rather than blocking on a prompt nobody will see. ask=permission_gate(_ask_operator) if args.interactive else None, - available_evals=offered if isinstance(offered, list) else None, - require_setup_contract=generic and not args.interactive, - max_turns=140 if generic and not args.interactive else 70, + **reading_settings(job, interactive=args.interactive), ) print(f"agent: {source.name} ({source.kind})") @@ -320,10 +312,7 @@ async def _understand(args: argparse.Namespace) -> int: interactive=args.interactive, until=lambda: load(destination) is not None or (destination / FAILURE_FILE).is_file(), - nudge=( - "Nothing was saved: you finished without calling submit_contract. Call it now " - "with the contract you worked out." - ), + nudge=SUBMIT_REMINDER, ) if (destination / FAILURE_FILE).is_file(): diff --git a/src/fi/alk/harness/framework_detection.py b/src/fi/alk/harness/framework_detection.py index c7650733c..d181bd75d 100644 --- a/src/fi/alk/harness/framework_detection.py +++ b/src/fi/alk/harness/framework_detection.py @@ -294,12 +294,14 @@ class _Work: every time. Python 3.12.3 re-decodes a token's whole line for each token, and one name can have thousands of targets: neither shows in the size of a file.""" - def __init__(self) -> None: + def __init__(self, limit: int | None = None) -> None: self._spent = 0 + # None follows _MAX_WORK; the agent scan gives each file its own, smaller limit. + self._limit = limit def charge(self, amount: int) -> None: self._spent += amount - if self._spent > _MAX_WORK: + if self._spent > (_MAX_WORK if self._limit is None else self._limit): raise _OverBudget() @@ -960,7 +962,11 @@ def detect_framework( *, repo_root: Path | None = None, package_dir: Path | None = None, + raise_faults: bool = False, ) -> dict[str, Any]: + """The folder's framework word. A fault in the check is ``unknown`` for a build; the agent + scan passes ``raise_faults=True`` to see it, so it can log the trace and mark the row + unfinished instead of reporting a finished ``unknown``.""" try: if folder is None or not Path(folder).is_dir(): logger.info("framework detection: name=%s blockers=%s", "unknown", []) @@ -978,15 +984,21 @@ def detect_framework( ) return result except Exception as exc: + if raise_faults: + raise logger.warning("framework detection failed: class=%s", type(exc).__name__) return {"name": "unknown", "version": None} -def _contract_workdir(source: Path, bundle_dir: Path) -> Path | None: +def _contract_workdir( + source: Path, bundle_dir: Path, contract_text: str | None = None +) -> Path | None: """`runtime.workdir` from the sealed contract, when it names a real inner folder.""" try: contract = json.loads( - (bundle_dir / "contract.json").read_text(encoding="utf-8") + contract_text + if contract_text is not None + else (bundle_dir / "contract.json").read_text(encoding="utf-8") ) runtime = contract.get("runtime") if isinstance(contract, dict) else None workdir = runtime.get("workdir") if isinstance(runtime, dict) else None @@ -1001,20 +1013,27 @@ def _contract_workdir(source: Path, bundle_dir: Path) -> Path | None: return None +def named_framework_folder( + source: Path, bundle_dir: Path, subfolder: str | None, *, contract_text: str | None = None +) -> Path | None: + """The folder the job names, else the sealed contract's own; ``None`` when neither does. + + ``contract_text`` is the contract already read, when the caller has read it. + """ + if subfolder: + return agent_folder(source, subfolder) + return _contract_workdir(source, bundle_dir, contract_text) + + def framework_folders_for( job: Any, manifest: Any, *, source: Path, bundle_dir: Path, work_directory: Path ) -> FrameworkFolders: runtime = manifest.runtime control_service = runtime.control_service if runtime.kind == "process" else None - subfolder = job.source.subfolder - folder = source - if subfolder: - folder = agent_folder(source, subfolder) - else: - workdir = _contract_workdir(source, bundle_dir) - if workdir is not None: - folder = workdir - elif control_service: + folder = named_framework_folder(source, bundle_dir, job.source.subfolder) + if folder is None: + folder = source + if control_service: for process in manifest.processes: if process.name == control_service: # The same base the runtime builds the process from. diff --git a/src/fi/alk/harness/reading_entry.py b/src/fi/alk/harness/reading_entry.py new file mode 100644 index 000000000..b94c74c7d --- /dev/null +++ b/src/fi/alk/harness/reading_entry.py @@ -0,0 +1,393 @@ +"""The one way into a reading of a repository: an agent read, or agents found, with no build. + +A reading is billed from ``cost.json`` and judged from ``failure.json``, so this module owns the +order in which those files appear. Bad input is refused before any model is paid for. The cost +file is on disk, at zero and marked incomplete, before the work starts, and only the work itself +may mark it complete. A hard time limit ends the process even when its own writes fail, because a +reading that outlives its sandbox leaves no file at all and is retried as a crash. +""" + +from __future__ import annotations + +import asyncio +import functools +import json +import logging +import math +import os +import threading +import typing +from collections.abc import Awaitable, Callable, Iterator +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from pydantic import BaseModel, ValidationError + +from . import spend +from .agent_type_check import EXIT_REFUSED +from .failure_file import write_failure_file +from .job import FailureDomain, HarnessFailure, HarnessJob, HarnessStage +from .outbound import redact_outbound_text + +READING_JOB_INVALID = "reading_job_invalid" +READING_OUTPUT_NOT_EMPTY = "reading_output_not_empty" +MESSAGE_LIMIT = 2000 +COST_FILE = "cost.json" + +EXIT_DONE = 0 +EXIT_RETRYABLE = 1 + +# Neither file is deleted by the hosted authoring entrypoint, so a read here takes nothing away +# from the code that loads them. +_SECRET_FILES = ( + Path("/run/futureagi/secrets.json"), + Path("/run/futureagi/simulator-secrets.json"), +) + +logger = logging.getLogger(__name__) +# Unmasked on purpose: the mask reads the secrets, and the trace of failing to read them holds none. +_secrets_logger = logging.getLogger(f"{__name__}.secrets") + + +@dataclass(frozen=True) +class ReadingMode: + name: str + check_job: Callable[[HarnessJob], HarnessFailure | None] + work: Callable[[HarnessJob, Path, Path], Awaitable[int]] + time_limit_seconds: float | None + timed_out_code: str + timed_out_retryable: bool + crashed_code: str + # When set, the limit is read from this variable as the reading starts, with + # ``time_limit_seconds`` as its default; the helper's own limit adds the margin to it. + limit_env: str | None = None + limit_margin_seconds: float = 0.0 + + +def limit_seconds(env: str, default: float) -> float: + """The seconds ``env`` names when they are a positive, finite number, else ``default``.""" + try: + seconds = float(os.environ.get(env, "")) + except ValueError: + return default + return seconds if math.isfinite(seconds) and seconds > 0 else default + + +def _hard_limit(mode: ReadingMode) -> float | None: + if mode.time_limit_seconds is None: + return None + base = ( + limit_seconds(mode.limit_env, mode.time_limit_seconds) + if mode.limit_env + else mode.time_limit_seconds + ) + return base + mode.limit_margin_seconds + + +@functools.cache +def _read_secret_values(paths: tuple[Path, ...]) -> tuple[str, ...]: + # Imported here: the module is large and the find-agents mode shares this helper. A broken + # import must not end a reading before its failure can be written, so it masks nothing extra. + try: + from .hosted_entrypoint import peek_secret_values + + values = {form for path in paths for value in peek_secret_values(path) for form in _forms(value)} + except Exception: + _secrets_logger.warning("secret values not read; only URL userinfo is masked", exc_info=True) + return () + # Longest first, so a whole value is masked before any of its parts. + return tuple(sorted(values, key=len, reverse=True)) + + +def _forms(value: str) -> Iterator[str]: + """The value, and the shapes a trace can show it in: each line, each string inside it when it + is JSON (a service account's key), and its escaped text as ``repr`` and JSON print it.""" + yield value + yield from value.splitlines() + yield repr(value)[1:-1] + yield json.dumps(value)[1:-1] + try: + parsed = json.loads(value) + except ValueError: + return + pending = [parsed] + while pending: + item = pending.pop() + if isinstance(item, str): + yield from _forms(item) + elif isinstance(item, dict): + pending.extend(item.values()) + elif isinstance(item, list): + pending.extend(item) + + +def _masked(text: str) -> str: + return redact_outbound_text(text, _read_secret_values(_SECRET_FILES)) + + +class MaskSecrets(logging.Filter): + """Masks a logger's lines, stack traces included, the way a reading's failure is masked.""" + + def filter(self, record: logging.LogRecord) -> bool: + record.msg, record.args = _masked(record.getMessage()), None + if record.exc_info: + # Kept only as masked text, so no handler can format the raw exception itself. + record.exc_text = _masked(logging.Formatter().formatException(record.exc_info)) + record.exc_info = None + return True + + +logger.addFilter(MaskSecrets()) + + +def write_reading_failure(failure: HarnessFailure, output: Path) -> Path | None: + """Leave ``failure`` for the platform with its message made safe to show; the first one stands. + + Details are written as given, so a mode never puts what a provider said in them. + """ + try: + # Masked before the cut, so a secret split by the cut can't survive half-matched. + message = _masked(failure.message)[:MESSAGE_LIMIT] + return write_failure_file(failure.model_copy(update={"message": message}), Path(output)) + except Exception: + logger.warning("failure file not written: code=%s", failure.code, exc_info=True) + return None + + +def write_reading_cost(output: Path, *, complete: bool) -> bool: + """Write the spend so far to ``cost.json``, and again after every turn; ``True`` when written. + + ``complete`` is kept only in this file, never in ``spend.snapshot()``, which builds share. + """ + try: + return spend.journal_to(Path(output) / COST_FILE, complete=complete) + except Exception: + # Accounting must never be the reason a reading fails. + logger.warning("cost file not written", exc_info=True) + return False + + +def reading_failure( + domain: FailureDomain, + code: str, + message: str, + *, + retryable: bool, + details: dict[str, Any] | None = None, +) -> HarnessFailure: + return HarnessFailure( + domain=domain, + stage=HarnessStage.UNDERSTANDING_AGENT, + code=code, + message=message, + retryable=retryable, + details=details or {}, + ) + + +def _models_in(annotation: Any) -> Iterator[type[BaseModel]]: + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + yield annotation + for inner in typing.get_args(annotation): + yield from _models_in(inner) + + +@functools.cache +def _job_field_names() -> frozenset[str]: + names: set[str] = set() + seen: set[type[BaseModel]] = set() + pending: list[type[BaseModel]] = [HarnessJob] + while pending: + model = pending.pop() + if model in seen: + continue + seen.add(model) + for name, field in model.model_fields.items(): + names.update(filter(None, (name, field.alias))) + pending.extend(_models_in(field.annotation)) + return frozenset(names) + + +def _unreadable_job(exc: Exception) -> str: + # Only where and how it failed: pydantic's own text repeats the job, and a job may hold a token. + if not isinstance(exc, ValidationError): + return type(exc).__name__ + fields = _job_field_names() + found = "; ".join( + # A dict's keys are the job's own data, and may be a token, so only field names and + # indexes are shown. + f"{'.'.join(str(part) if isinstance(part, int) or part in fields else '*' for part in error['loc']) or 'job'}: {error['type']}" + for error in exc.errors(include_input=False) + ) + return f"ValidationError: {found}" + + +def _output_in_use(output: Path) -> bool: + try: + if not os.path.lexists(output): + return False + return not output.is_dir() or next(output.iterdir(), None) is not None + except OSError: + # A folder that can't be listed can't be shown to be empty. + return True + + +class _HardLimit: + """Ends the process at the limit. Before the work returns, a timeout is written first; after + it, the work's files stand and the process ends with the work's own code.""" + + def __init__(self, mode: ReadingMode, output: Path, limit: float) -> None: + self._mode = mode + self._output = output + self._limit = limit + self._returned: int | None = None + self._timer = threading.Timer(limit, self._end_the_process) + self._timer.daemon = True + self._timer.start() + + def returned(self, code: int) -> None: + self._returned = code + + def cancel(self) -> None: + self._timer.cancel() + + def _end_the_process(self) -> None: + returned = self._returned + if returned is not None: + # Only a thread the work left behind keeps the process alive now. Its files are on + # disk, so nothing is written over them. + os._exit(returned) + mode, output, limit = self._mode, self._output, self._limit + exit_code = EXIT_RETRYABLE if mode.timed_out_retryable else EXIT_REFUSED + # Nothing logs on the way out: a blocked stderr must not hold off the exit. + logging.disable(logging.CRITICAL) + try: + write_reading_failure( + reading_failure( + FailureDomain.INFRASTRUCTURE, + mode.timed_out_code, + f"The {mode.name} reading ran past its {limit:g} s limit " + "and was stopped.", + retryable=mode.timed_out_retryable, + ), + output, + ) + write_reading_cost(output, complete=False) + finally: + # Work running in a thread can't be cancelled, and a normal exit would wait for it. + os._exit(exit_code) + + +async def _work_then_stop_the_clock( + mode: ReadingMode, job: HarnessJob, source: Path, output: Path, limit: _HardLimit | None +) -> int: + code = await mode.work(job, source, output) + # Told inside the loop: asyncio.run waits for leftover worker threads before it returns, and + # the limit must not write a timeout over a reading that has already ended. + if limit is not None: + limit.returned(code) + return code + + +def run_reading_mode(mode: ReadingMode, *, job_path: Path, source: Path, output: Path) -> int: + """Refuse what can't be read, start the cost at zero, then run the mode's work.""" + # Read afresh for each reading, so one job's secrets never stand in for another's. + _read_secret_values.cache_clear() + _read_secret_values(_SECRET_FILES) + output = Path(output) + + try: + job = HarnessJob.model_validate(json.loads(Path(job_path).read_text(encoding="utf-8"))) + except (OSError, ValueError, RecursionError) as exc: + write_reading_failure( + reading_failure( + FailureDomain.PLATFORM_SYNC, + READING_JOB_INVALID, + f"The job file could not be read as a harness job ({_unreadable_job(exc)}).", + retryable=False, + ), + output, + ) + return EXIT_REFUSED + + try: + refused = mode.check_job(job) + except Exception as exc: + logger.warning("the %s job check crashed", mode.name, exc_info=True) + # Nothing has been spent yet, so no cost file: a broken check is a crash, not a refusal. + write_reading_failure( + reading_failure( + FailureDomain.INFRASTRUCTURE, + mode.crashed_code, + f"{type(exc).__name__}: {exc}", + retryable=True, + ), + output, + ) + return EXIT_RETRYABLE + if refused is not None: + write_reading_failure(refused, output) + return EXIT_REFUSED + + # After the job checks on purpose: a stale folder is refused either way, and the failure + # they wrote into it is no less true. + if _output_in_use(output): + write_reading_failure( + reading_failure( + FailureDomain.PLATFORM_SYNC, + READING_OUTPUT_NOT_EMPTY, + "The reading's output folder already holds files; give it an empty folder.", + retryable=False, + ), + output, + ) + return EXIT_REFUSED + + if not write_reading_cost(output, complete=False): + # A model paid for while its cost can't be recorded would read as nothing spent. + write_reading_failure( + reading_failure( + FailureDomain.INFRASTRUCTURE, + mode.crashed_code, + "The reading could not write its cost file, so it stopped before any model ran.", + retryable=True, + ), + output, + ) + return EXIT_RETRYABLE + + # Read now, not when the mode was made, so it is the same limit the work reads as it starts. + seconds = _hard_limit(mode) + limit = None if seconds is None else _HardLimit(mode, output, seconds) + crash: BaseException | None = None + try: + logger.info("reading started at %s", datetime.now(timezone.utc).isoformat()) + code = asyncio.run(_work_then_stop_the_clock(mode, job, source, output, limit)) + except BaseException as exc: + # A cancel or a group of them leaking out of the work is a crash too; an interrupt or an + # exit is let through. + if isinstance(exc, (KeyboardInterrupt, SystemExit)): + raise + logger.error("the %s reading crashed", mode.name, exc_info=True) + crash = exc + finally: + # Cancelled first, so the limit can't fire during a crash's writes or the caller's cleanup. + if limit is not None: + limit.cancel() + + if crash is not None: + write_reading_failure( + reading_failure( + FailureDomain.INFRASTRUCTURE, + mode.crashed_code, + f"{type(crash).__name__}: {crash}", + retryable=True, + ), + output, + ) + write_reading_cost(output, complete=False) + return EXIT_RETRYABLE + # The work wrote its own final cost; rewriting it here could mark a lost turn as paid. + return code diff --git a/src/fi/alk/harness/reading_result.py b/src/fi/alk/harness/reading_result.py new file mode 100644 index 000000000..0927f59a4 --- /dev/null +++ b/src/fi/alk/harness/reading_result.py @@ -0,0 +1,267 @@ +"""What a finished reading leaves: the agent's description from its saved contract, or why not. + +A saved contract has been paid for, so once one exists nothing after it is retried. Without one, +the reading's own turns decide whether a retry could get further. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from pydantic import ValidationError + +from .agent_type_check import EXIT_REFUSED +from .backends import StageDone +from .failure_file import FAILURE_FILE +from .framework_detection import detect_framework, named_framework_folder +from .job import FailureDomain, HarnessJob +from .reading_entry import ( + EXIT_DONE, + EXIT_RETRYABLE, + MaskSecrets, + reading_failure, + write_reading_failure, +) +from .scenario_source import authored_agent_document + +if TYPE_CHECKING: + from .agent_reading import _ReadingStage + +logger = logging.getLogger(__name__) +logger.addFilter(MaskSecrets()) + +AGENT_FILE = "agent.json" +CONTRACT_FILE = "contract.json" + +AGENT_READING_MODEL_ERROR = "agent_reading_model_error" +AGENT_READING_INCOMPLETE = "agent_reading_incomplete" +AGENT_READING_TOO_LARGE = "agent_reading_too_large" +AGENT_DOCUMENT_INVALID = "agent_document_invalid" + +# The platform's own caps, compared the way it compares them. +_DOCUMENT_LIMIT_BYTES = 1_048_576 +_CONTRACT_LIMIT_BYTES = 5 * 1_048_576 + +# A model out of turns can't save, so a reminder after one only adds cost. +_TURN_LIMITS = frozenset({"max_turns", "error_max_turns"}) +# Matched only, never repeated in a message: these are ADK's words, not ours. +_CALL_LIMIT = re.compile(r"Max number of llm calls limit of `\d+` exceeded") +# Statuses a retry may get past; any other 4xx would only be refused again. +_PASSING_STATUSES = frozenset({408, 409, 429}) + + +def saved_file(output: Path) -> str: + if (output / FAILURE_FILE).is_file(): + return FAILURE_FILE + if (output / CONTRACT_FILE).is_file(): + return CONTRACT_FILE + return "" + + +def out_of_turns(done: StageDone) -> bool: + # ADK enforces its call budget by raising, which the Gemini backend reports as a failure with + # no status, so ADK's own words are the signal. Matched whole, since the phrase inside other + # text may be the model's own. The count is no signal: it catches no real limit, and it would + # read a request cut on the last allowed call as one. + return done.outcome in _TURN_LIMITS or ( + done.is_error + and done.api_error_status is None + and any(_CALL_LIMIT.fullmatch(str(said)) for said in done.errors) + ) + + +def _refusing(status: Any) -> bool: + return ( + isinstance(status, int) + and not isinstance(status, bool) + and 400 <= status <= 499 + and status not in _PASSING_STATUSES + ) + + +def model_error(output: Path, message: str, status: Any = None, *, retryable: bool) -> int: + write_reading_failure( + reading_failure( + FailureDomain.INFRASTRUCTURE, + AGENT_READING_MODEL_ERROR, + message, + retryable=retryable, + # Only our own reading of the status: what the provider said stays in the message. + details={"status": status if isinstance(status, int) else None}, + ), + output, + ) + return EXIT_RETRYABLE if retryable else EXIT_REFUSED + + +def classify( + job: HarnessJob, + source: Path, + output: Path, + stage: _ReadingStage, + error: BaseException | None, +) -> int: + saved = saved_file(output) + if saved == FAILURE_FILE: + return EXIT_REFUSED + # Before any error: the gate accepted the reading, and refusing it now would pay for it twice. + if saved == CONTRACT_FILE: + return document(job, source, output) + forbidden = stage.forbidden_models() + if forbidden: + # A retry would bill the same model again. + return model_error( + output, + "The provider billed a model this harness may not spend on (" + + ", ".join(sorted(forbidden)) + + "), so the reading was stopped.", + retryable=False, + ) + # A last turn out of turns spent a whole budget, so a retry would likely spend another and + # stop the same way, whatever failed before it. + ran_out = bool(stage.done) and out_of_turns(stage.done[-1]) + failed = [ + done + for done in stage.done + if done.api_error_status is not None + or (done.is_error and not out_of_turns(done)) + ] + if not ran_out and (failed or error is not None): + status = next( + (done.api_error_status for done in reversed(failed) if done.api_error_status is not None), + None, + ) + said = [str(one) for done in failed for one in done.errors] + if error is not None: + said.append(f"{type(error).__name__}: {error}") + shown = f" (status {status})" if isinstance(status, int) else "" + return model_error( + output, + f"The model provider failed while the agent was being read{shown}: " + + ("; ".join(said) or "no detail was given."), + status, + retryable=not _refusing(status), + ) + write_reading_failure( + reading_failure( + FailureDomain.INFRASTRUCTURE, + AGENT_READING_INCOMPLETE, + "The model finished without saving the agent's description.", + retryable=False, + details={ + "outcome": stage.done[-1].outcome if stage.done else "", + "turns": sum(int(done.turns or 0) for done in stage.done), + }, + ), + output, + ) + return EXIT_REFUSED + + +class _TooLarge(Exception): + def __init__(self, sizes: dict[str, int]) -> None: + super().__init__("too large") + self.sizes = sizes + + +def _framework_folder(job: HarnessJob, source: Path, output: Path, contract_text: str) -> Path: + # As a build chooses it when there is no build: the named folder, else the contract's own. + return ( + named_framework_folder( + source, output, job.source.subfolder, contract_text=contract_text + ) + or source + ) + + +def _write_whole(path: Path, body: bytes) -> None: + # Written whole, then moved: the platform must never read half a document. + handle = tempfile.NamedTemporaryFile( + "wb", dir=path.parent, prefix=".agent-", suffix=".tmp", delete=False + ) + try: + with handle as writing: + writing.write(body) + os.replace(handle.name, path) + except BaseException: + Path(handle.name).unlink(missing_ok=True) + raise + + +def document(job: HarnessJob, source: Path, output: Path) -> int: + """The agent's description from the saved contract, as a build makes it.""" + # One catch over every step: a saved contract is paid for, so nothing after it is retryable. + try: + # Read once, here, so what is checked is what is used: the session may still save while + # the framework scan below runs. + with (output / CONTRACT_FILE).open("rb") as handle: + raw = handle.read(_CONTRACT_LIMIT_BYTES + 1) + contract_bytes = max(os.fstat(handle.fileno()).st_size, len(raw)) + if contract_bytes > _CONTRACT_LIMIT_BYTES: + raise _TooLarge({"contract_bytes": contract_bytes}) + contract_text = raw.decode("utf-8") + framework = detect_framework( + _framework_folder(job, source, output, contract_text), + repo_root=source, + package_dir=None, + ) + # The build's own document, made by the build's own code. + made = authored_agent_document( + contract_text, subfolder=job.source.subfolder, framework=framework + ) + body = json.dumps(made, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + stored = json.dumps(made["agent"], ensure_ascii=False).encode("utf-8") + if len(body) > _DOCUMENT_LIMIT_BYTES or len(stored) > _DOCUMENT_LIMIT_BYTES: + raise _TooLarge( + { + "agent_bytes": len(body), + "stored_bytes": len(stored), + "contract_bytes": contract_bytes, + } + ) + _write_whole(output / AGENT_FILE, body) + except _TooLarge as too_large: + write_reading_failure( + reading_failure( + FailureDomain.INFRASTRUCTURE, + AGENT_READING_TOO_LARGE, + "The agent's description is larger than the platform stores (1 MiB for the " + "description, 5 MiB for the contract).", + retryable=False, + details=dict(too_large.sizes), + ), + output, + ) + return EXIT_REFUSED + except Exception as exc: + code = str(getattr(exc, "code", "") or "") + logger.warning( + "agent document not built: class=%s code=%s", + type(exc).__name__, + code, + # pydantic's trace repeats values from the contract, which the mask can't know. + exc_info=not isinstance(exc, ValidationError), + ) + details = {"class": type(exc).__name__, "code": code} + if getattr(exc, "detail", "") == "subfolder_mismatch": + # Named as the build names it: the one detail that is ours, not the contract's. + details["detail"] = "subfolder_mismatch" + write_reading_failure( + reading_failure( + FailureDomain.INFRASTRUCTURE, + AGENT_DOCUMENT_INVALID, + "The agent's description could not be built from its saved contract " + f"({type(exc).__name__}{': ' + code if code else ''}).", + retryable=False, + details=details, + ), + output, + ) + return EXIT_REFUSED + return EXIT_DONE diff --git a/src/fi/alk/harness/scan_ai.py b/src/fi/alk/harness/scan_ai.py new file mode 100644 index 000000000..d28b938f2 --- /dev/null +++ b/src/fi/alk/harness/scan_ai.py @@ -0,0 +1,981 @@ +"""Asks a model where a repository's agents are, when the plain code check found none it can trust. + +The customer pays for every turn here, so the step is shaped around the bill. It says one thing, +once: a second message in a Claude session re-reports the first one's cost, and a reminder after +a turn limit is a fresh paid turn. An answer the model has submitted is kept whatever happens +after it, because it was paid for. The time limit always ends in the same stop sequence, so the +cost of a call in flight is either recorded or marked incomplete, never silently dropped. + +The model reads the repository through three tools of this module's own, not the kit's +general file tools. Those search with a regular expression over whole files with no bound, +which one minified line or one named pipe turns into a stalled event loop. These search for +plain text, bound every call by entries, bytes, characters and seconds, and never show a file +the kit treats as a secret. +""" + +from __future__ import annotations + +import asyncio +import fnmatch +import logging +import os +import re +import stat +import threading +from collections.abc import Callable, Iterator +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Literal + +from .backends import SessionSpec, StageDone, ToolServer, ToolSpec +from .backends.files import SECRET_REFUSAL, above_secrets, secret_path +from .config import chosen_model, load_skill +from .framework_detection import _ALWAYS_SKIPPED, _TOP_ONLY_SKIPS +from .reading_entry import MaskSecrets +from .scan_walk import fits_path_rule +from .session import Stage, StageIdleTimeout, Turn + +logger = logging.getLogger(__name__) +logger.addFilter(MaskSecrets()) + +STAGE_NAME = "find_agents" +MAX_TURNS = 40 +IDLE_SECONDS = 90.0 +OPEN_SECONDS = 30.0 +# The stop sequence. Long enough for a provider to send its last bill, short enough that a +# stuck one can't eat the time the rest of the scan needs. +INTERRUPT_SECONDS = 10.0 +SPEND_WAIT_SECONDS = 60.0 +CANCEL_SECONDS = 5.0 +EXIT_SECONDS = 10.0 +TOOL_SECONDS = 20.0 + +MAX_PAIRS = 50 +MAX_ENTRIES = 2_000 +MAX_READ_BYTES = 33_554_432 +MAX_FILE_BYTES = 1_048_576 +MAX_LINE_BYTES = 65_536 +MAX_RESULT_CHARS = 8_000 +MAX_READ_LINES = 400 +# As the kit's general Read cuts a line, so a minified file can't fill a result with one line. +MAX_LINE_CHARS = 2_000 +HIT_CHARS = 300 +MAX_PATTERN_CHARS = 200 +MAX_PATH_CHARS = 4_096 +MAX_PATH_PARTS = 256 + +STOPPED_EARLY = "Stopped early at the search limit: results are incomplete. Narrow the path." +TOOLS_AFTER_SUBMIT = "Already submitted: end your turn." +# ADK's own message when a run spends its call budget, matched whole so that the phrase quoted +# inside some other error never reads as a turn limit. +_CALL_LIMIT = re.compile(r"Max number of llm calls limit of `\d+` exceeded") +SUBMIT_AFTER_SUBMIT = "Already submitted." + +_OUTSIDE = "That path is not allowed: give a path inside the repository." +_SKIPPED = "That path is in a folder this step skips: installed, built or generated files." +_MISSING = "No such file or folder in the repository." +_NOT_A_FILE = "That is not a file this tool reads: only regular files up to 1 MiB are read." +_NOT_A_FOLDER = "That is not a folder in the repository." +_BAD_NUMBER = "offset and limit must be whole numbers." +_BAD_PATTERN = "pattern must be text of 1 to 200 characters." +_BAD_TEXT = "text must be plain text of 1 to 200 characters." +_BAD_GLOB = "glob must be text of at most 200 characters." +_BAD_IGNORE_CASE = "ignore_case must be true or false." +_BAD_SUBMIT = 'Give {"agents": [{"folder": ..., "entry": ...}]} with at most 50 agents.' +_TOO_SLOW = "That call took too long and was stopped. Narrow the path." +_FAILED = "That call failed. Try a different path or argument." + +# Names the kit's secret filter doesn't cover. A source file named like `secret_manager.py` is +# hidden too; that's the price of never showing a settings file under an unusual name. +_SECRET_NAMES = ( + "*.env", + "*.env.*", + "*secret*", + "*credential*", + "id_rsa*", + "id_ed25519*", + "id_ecdsa*", + "*.tfvars", +) + +_OPENING = ( + "Find the runnable AI agents in this repository and submit them with " + "mcp__scan__submit_agents, as your instructions say. A plain code check found none outside " + "test and example folders, apart from any files that only call a model, which it has " + "already listed. What it counted: " +) + + +@dataclass(frozen=True) +class AiOutcome: + kind: Literal["pairs", "empty", "no_answer", "timed_out", "model_error", "forbidden"] + pairs: tuple[tuple[str, str], ...] + refused: int + complete: bool + retryable: bool + message: str + # Real files the model named whose path the platform's rule can't carry, counted once each + # over every submit: they are left out, so the list is not whole. + uncarried: int = 0 + + +class _Refused(Exception): + """A tool call the tool won't answer, with the fixed words the model is told.""" + + def __init__(self, message: str) -> None: + super().__init__(message) + self.message = message + + +class _ToolFailed(Exception): + pass + + +@dataclass +class _Answer: + """What the model has submitted. Shared by the submit tool and the file tools.""" + + called: bool = False + pairs: tuple[tuple[str, str], ...] = () + refused: int = 0 + uncarried: set[str] = field(default_factory=set) + + def finished(self) -> bool: + return bool(self.pairs) + + +class _ScanStage(Stage): + """A stage that keeps its one turn's raw end and says when a turn's cost is known.""" + + def __init__( + self, spec: SessionSpec, *, name: str, on_turn_end: Callable[[], None] + ) -> None: + # A replayed quiet stream would be a second paid opening, and its first spend is + # never recorded. + super().__init__(spec, name=name, idle_retries=0) + self.done: list[StageDone] = [] + self._on_turn_end = on_turn_end + + def _events(self, received: Any, turn: Turn) -> list[Any]: + try: + return super()._events(received, turn) + finally: + # In a finally because the parent raises for a forbidden model after recording + # its spend, and that turn's cost still has to be noted. + if isinstance(received, StageDone): + self.done.append(received) + try: + self._on_turn_end() + except Exception: + # A failed cost note must not turn a paid answer into a retried one. The + # trace is logged (secret values masked), so a fault in the cost writer can be found. + logger.warning("find_agents: the cost note after the turn failed", exc_info=True) + + +# -- the run ----------------------------------------------------------------------------------- + + +async def run_ai_step( + source: Path, *, hints: str, seconds: float, on_turn_end: Callable[[], None] +) -> AiOutcome: + """Ask the model once for the repository's agents, within ``seconds``.""" + answer = _Answer() + spec = SessionSpec( + system_prompt=load_skill("find-agents"), + servers={ + "scan_files": rooted_file_tools(source, finished=answer.finished), + "scan": _submit_server(source, answer), + }, + builtins=(), + cwd=str(source), + max_turns=MAX_TURNS, + model=chosen_model(), + ask=None, + gated=True, + thinking=False, + idle_timeout_seconds=IDLE_SECONDS, + ) + stage = _ScanStage(spec, name=STAGE_NAME, on_turn_end=on_turn_end) + try: + outcome = await _run(stage, answer, hints, seconds) + finally: + await _leave(stage) + logger.info( + "find_agents: %s, %d kept, %d refused, complete=%s", + outcome.kind, + len(outcome.pairs), + outcome.refused, + outcome.complete, + ) + return outcome + + +async def _run(stage: _ScanStage, answer: _Answer, hints: str, seconds: float) -> AiOutcome: + try: + await asyncio.wait_for(stage.__aenter__(), OPEN_SECONDS) + except Exception as exc: + logger.warning("find_agents: the model session did not open", exc_info=True) + return AiOutcome( + "model_error", + (), + 0, + False, + True, + f"the model session did not open ({type(exc).__name__})", + ) + task = asyncio.create_task(stage.say(_opening(hints))) + try: + finished, _ = await asyncio.wait({task}, timeout=seconds) + stopped = not finished + if stopped: + await _stop(stage, task) + failure = None if not task.done() or task.cancelled() else task.exception() + finally: + if not task.done(): + task.cancel() + return _outcome(stage, answer, stopped=stopped, failure=failure) + + +def _opening(hints: str) -> str: + return _OPENING + (hints or "nothing.") + + +async def _stop(stage: _ScanStage, task: asyncio.Task[Any]) -> None: + try: + await asyncio.wait_for(stage.interrupt_response(), INTERRUPT_SECONDS) + except asyncio.TimeoutError: + logger.warning("find_agents: the interrupt got no answer in %ss", INTERRUPT_SECONDS) + except Exception: + # The session may already be gone; the wait for its bill below still runs. + logger.warning("find_agents: the interrupt failed", exc_info=True) + # The turn's bill arrives after the interrupt, if it arrives at all. + await asyncio.wait({task}, timeout=SPEND_WAIT_SECONDS) + if not task.done(): + task.cancel() + await asyncio.wait({task}, timeout=CANCEL_SECONDS) + + +async def _leave(stage: _ScanStage) -> None: + try: + await asyncio.wait_for(stage.__aexit__(None, None, None), EXIT_SECONDS) + except asyncio.TimeoutError: + logger.warning("find_agents: the session did not close in %ss", EXIT_SECONDS) + except Exception: + # The answer is already kept; a session that fails to close doesn't change it. + logger.warning("find_agents: the session failed to close", exc_info=True) + + +def _outcome( + stage: _ScanStage, answer: _Answer, *, stopped: bool, failure: BaseException | None +) -> AiOutcome: + complete = bool(stage.done) and not stopped + if answer.pairs: + return AiOutcome( + "pairs", answer.pairs, answer.refused, complete, False, "", len(answer.uncarried) + ) + if stage.forbidden_models(): + return AiOutcome( + "forbidden", + (), + 0, + complete, + False, + "the provider billed a model this kit may not spend on", + ) + if stopped: + return AiOutcome( + "timed_out", (), 0, complete, False, "the model step ran out of time" + ) + if isinstance(failure, StageIdleTimeout): + return AiOutcome( + "model_error", + (), + 0, + complete, + True, + "the model stream went quiet (StageIdleTimeout)", + ) + if failure is not None: + return AiOutcome( + "model_error", + (), + 0, + complete, + True, + f"the model step failed ({type(failure).__name__})", + ) + done = stage.done[-1] if stage.done else None + if done is not None and done.api_error_status: + status = done.api_error_status + said = ( + f" with status {status}" + if isinstance(status, int) and not isinstance(status, bool) + else "" + ) + return AiOutcome( + "model_error", + (), + 0, + complete, + not _client_error(status), + f"the model call failed{said}", + ) + if done is not None and done.is_error and not _turn_limit(done): + return AiOutcome( + "model_error", (), 0, complete, True, "the model call ended in an error" + ) + if answer.called: + return AiOutcome("empty", (), 0, complete, False, "", len(answer.uncarried)) + return AiOutcome( + "no_answer", + (), + 0, + complete, + False, + "the model ended without submitting an answer", + ) + + +def _turn_limit(done: StageDone) -> bool: + if done.outcome in ("error_max_turns", "max_turns"): + return True + if not done.is_error or done.api_error_status: + return False + # Vertex-Gemini's call limit arrives as ADK's exception: failed, no status. The turn count + # can't say it: a reply with no parts isn't counted, and a status-less cut on the last allowed + # request reaches the limit without being one. ADK's fixed words are only matched, never + # repeated. + return any(_CALL_LIMIT.fullmatch(str(error)) for error in done.errors) + + +def _client_error(status: Any) -> bool: + # A request the provider refuses won't be accepted on a retry either, except for these + # three, which say "not now" rather than "not this". + return ( + isinstance(status, int) + and not isinstance(status, bool) + and 400 <= status <= 499 + and status not in (408, 409, 429) + ) + + +# -- the tools --------------------------------------------------------------------------------- + + +def _ok(text: str) -> dict[str, Any]: + return {"content": [{"type": "text", "text": text}]} + + +def _error(text: str) -> dict[str, Any]: + return {"content": [{"type": "text", "text": text}], "is_error": True} + + +async def _in_thread(work: Callable[..., Any], *args: Any) -> Any: + """``work(*args)`` in a daemon thread, so a stalled disk can hold neither the loop nor exit.""" + loop = asyncio.get_running_loop() + answer = loop.create_future() + + def settle(result: Any, error: BaseException | None) -> None: + # The wait below may have given up and cancelled the future already. + if answer.done(): + return + if error is None: + answer.set_result(result) + else: + answer.set_exception(error) + + def run() -> None: + result: Any = None + error: BaseException | None = None + try: + result = work(*args) + except _Refused as refused: + error = refused + except BaseException as exc: + error = _ToolFailed() + # Kept as the cause, so the logged trace shows the worker's own fault. + error.__cause__ = exc + try: + loop.call_soon_threadsafe(settle, result, error) + except RuntimeError: + pass + + threading.Thread(target=run, name="find-agents-tool", daemon=True).start() + return await asyncio.wait_for(answer, TOOL_SECONDS) + + +async def _answered(work: Callable[..., str], *args: Any) -> dict[str, Any]: + try: + return _ok(await _in_thread(work, *args)) + except _Refused as refused: + return _error(refused.message) + except asyncio.TimeoutError: + return _error(_TOO_SLOW) + except Exception: + # The model is told only that the call failed; the trace is logged, secret values masked. + logger.warning("find_agents: a tool call failed", exc_info=True) + return _error(_FAILED) + + +def rooted_file_tools( + root: Path, *, finished: Callable[[], bool] | None = None +) -> ToolServer: + """``read``, ``glob`` and ``grep``, confined to ``root`` and bounded on every call. + + ``finished`` says the answer is in; from then on the tools read nothing. ``None`` means + the answer is never in. + """ + base = os.path.realpath(str(root)) + + async def guarded(work: Callable[[str, dict[str, Any]], str], arguments: Any) -> dict[str, Any]: + try: + if finished is not None and finished(): + return _error(TOOLS_AFTER_SUBMIT) + return await _answered(work, base, arguments) + except Exception: + logger.warning("find_agents: a tool call failed", exc_info=True) + return _error(_FAILED) + + # Each looks its worker up when called, not when built. + async def read(arguments: dict[str, Any]) -> dict[str, Any]: + return await guarded(_read_lines, arguments) + + async def glob(arguments: dict[str, Any]) -> dict[str, Any]: + return await guarded(_glob_files, arguments) + + async def grep(arguments: dict[str, Any]) -> dict[str, Any]: + return await guarded(_grep_files, arguments) + + return ToolServer( + name="scan_files", + tools=[ + ToolSpec( + name="read", + description=( + "Read one file of the repository. Arguments: path (relative to the " + "repository's root), optional offset (first line, from 1) and limit (at " + "most 400 lines)." + ), + input_schema={ + "type": "object", + "properties": { + "path": {"type": "string"}, + "offset": {"type": "integer"}, + "limit": {"type": "integer"}, + }, + "required": ["path"], + }, + handler=read, + ), + ToolSpec( + name="glob", + description=( + "List the repository's files whose path or name matches a pattern such as " + "*.py or **/agent*.ts. Arguments: pattern, optional path of a folder to " + "look under." + ), + input_schema={ + "type": "object", + "properties": { + "pattern": {"type": "string"}, + "path": {"type": "string"}, + }, + "required": ["pattern"], + }, + handler=glob, + ), + ToolSpec( + name="grep", + description=( + "Find lines containing text, matched exactly as written: plain text, not a " + "regular expression. Arguments: text, optional path (a file or folder), glob " + "(file names to search, such as *.ts) and ignore_case." + ), + input_schema={ + "type": "object", + "properties": { + "text": {"type": "string"}, + "path": {"type": "string"}, + "glob": {"type": "string"}, + "ignore_case": {"type": "boolean"}, + }, + "required": ["text"], + }, + handler=grep, + ), + ], + ) + + +def _secret_name(name: str) -> bool: + lowered = name.lower() + return any(fnmatch.fnmatchcase(lowered, pattern) for pattern in _SECRET_NAMES) + + +def _skipped_folder(name: str, *, at_top: bool) -> bool: + # The same folders the framework check skips, so the model and the code look at one tree. + return name in _ALWAYS_SKIPPED or (at_top and name in _TOP_ONLY_SKIPS) + + +def _in_skipped_folder(folders: list[str]) -> bool: + return any( + _skipped_folder(name, at_top=index == 0) for index, name in enumerate(folders) + ) + + +def _usable(status: os.stat_result) -> bool: + # A hard link can make a file outside the source look like one inside it. + return ( + stat.S_ISREG(status.st_mode) + and status.st_nlink == 1 + and status.st_size <= MAX_FILE_BYTES + ) + + +def _under(path: str, root: str) -> bool: + return path == root or path.startswith(root.rstrip(os.sep) + os.sep) + + +def _place(root: str, raw: Any) -> tuple[str, str]: + """The model's ``path`` as (the path as written, joined to the root; where it resolves).""" + if raw is None or raw == "": + raw = "." + if ( + not isinstance(raw, str) + or "\0" in raw + or len(raw) > MAX_PATH_CHARS + or len(raw.split("/")) > MAX_PATH_PARTS + ): + raise _Refused(_OUTSIDE) + joined = raw if os.path.isabs(raw) else os.path.join(root, raw) + real = os.path.realpath(joined) + if not _under(real, root): + raise _Refused(_OUTSIDE) + if secret_path(raw, Path(root)) or above_secrets(raw, Path(root)): + raise _Refused(SECRET_REFUSAL) + if real != root: + parts = os.path.relpath(real, root).split(os.sep) + # Refused exactly where the walk would never have shown it. + if any(_secret_name(part) for part in parts): + raise _Refused(SECRET_REFUSAL) + if _in_skipped_folder(parts): + raise _Refused(_SKIPPED) + return joined, real + + +def _hidden(path: str, name: str) -> bool: + return _secret_name(name) or secret_path(path) + + +@dataclass +class _Budget: + entries: int = 0 + read_bytes: int = 0 + stopped: bool = False + + +def _listed(folder: str, budget: _Budget) -> list[str]: + names: list[str] = [] + with os.scandir(folder) as entries: + for entry in entries: + if budget.entries >= MAX_ENTRIES: + budget.stopped = True + break + budget.entries += 1 + names.append(entry.name) + return sorted(names) + + +def _walk(root: str, start: str, budget: _Budget) -> Iterator[tuple[str, str]]: + """Every usable file under ``start``, as (its path, its path relative to ``root``).""" + stack = [start] + while stack and not budget.stopped: + folder = stack.pop() + try: + names = _listed(folder, budget) + except OSError: + continue + below: list[str] = [] + for name in names: + path = os.path.join(folder, name) + try: + status = os.lstat(path) + except OSError: + continue + if _hidden(path, name): + continue + if stat.S_ISDIR(status.st_mode): + if not _skipped_folder(name, at_top=folder == root): + below.append(path) + elif _usable(status) and not name.lower().endswith(".min.js"): + yield path, os.path.relpath(path, root) + stack.extend(reversed(below)) + + +def _read_bytes(path: str) -> bytes: + # O_NONBLOCK so that a file swapped for a named pipe since its lstat can't hang the thread. + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + if not _usable(os.fstat(descriptor)): + raise _Refused(_NOT_A_FILE) + chunks: list[bytes] = [] + remaining = MAX_FILE_BYTES + 1 + while remaining > 0: + chunk = os.read(descriptor, remaining) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + finally: + os.close(descriptor) + if len(data) > MAX_FILE_BYTES: + raise _Refused(_NOT_A_FILE) + return data + + +def _one_file(joined: str) -> None: + try: + status = os.lstat(joined) + except FileNotFoundError: + raise _Refused(_MISSING) from None + if not _usable(status) or os.path.basename(joined).lower().endswith(".min.js"): + raise _Refused(_NOT_A_FILE) + + +def _whole(value: Any, default: int) -> int: + if value is None: + return default + if isinstance(value, int) and not isinstance(value, bool): + return value + if isinstance(value, str) and value.isdecimal() and len(value) <= 9: + return int(value) + raise _Refused(_BAD_NUMBER) + + +def _text(value: Any, *, required: bool, problem: str) -> str: + if value is None and not required: + return "" + if not isinstance(value, str) or len(value) > MAX_PATTERN_CHARS or (required and not value): + raise _Refused(problem) + return value + + +def _finished_text(lines: list[str], *, stopped: bool, nothing: str) -> str: + body = "\n".join(lines) + if stopped: + return f"{body}\n{STOPPED_EARLY}" if body else STOPPED_EARLY + return body or nothing + + +# Room for the note, so a capped result still ends with it inside the limit. +_ROOM = MAX_RESULT_CHARS - len(STOPPED_EARLY) - 1 + + +def _read_lines(root: str, arguments: dict[str, Any]) -> str: + joined, real = _place(root, arguments.get("path")) + if real == root: + raise _Refused(_NOT_A_FILE) + offset = max(_whole(arguments.get("offset"), 1), 1) + limit = min(max(_whole(arguments.get("limit"), MAX_READ_LINES), 1), MAX_READ_LINES) + _one_file(joined) + lines = _read_bytes(joined).splitlines() + if not lines: + return "(empty file)" + if offset > len(lines): + return f"(the file has {len(lines)} lines; start at a lower offset)" + shown: list[str] = [] + size = 0 + number = offset + for raw in lines[offset - 1 : offset - 1 + limit]: + line = f"{number}\t{raw.decode('utf-8', 'replace')[:MAX_LINE_CHARS]}" + # Room for the continuation line and the note. + if size + len(line) + 1 > _ROOM - 80: + shown.append(f"... call again with offset={number}") + return _finished_text(shown, stopped=True, nothing="") + shown.append(line) + size += len(line) + 1 + number += 1 + remaining = len(lines) - (number - 1) + if remaining > 0: + shown.append(f"... ({remaining} more lines; call again with offset={number})") + return "\n".join(shown) + + +_BRACE = re.compile(r"\{([^{}]*,[^{}]*)\}") +_MAX_FORMS = 32 + + +def _pattern_forms(pattern: str) -> tuple[str, ...]: + """The pattern as a shell reads it: fnmatch has no ``{a,b}``, and its ``**/`` needs a folder + in front, so ``**/package.json`` would miss the one at the root. Never more than 32 forms.""" + forms, done = [pattern], [] + while forms and len(done) + len(forms) <= _MAX_FORMS: + form = forms.pop() + brace = _BRACE.search(form) + if brace is None: + done.append(form) + continue + forms.extend( + form[: brace.start()] + choice + form[brace.end() :] + for choice in brace.group(1).split(",") + ) + done.extend(forms) + grown = [] + for form in done[:_MAX_FORMS]: + grown.append(form) + flat = form.replace("/**/", "/") + while flat.startswith("**/"): + flat = flat[3:] + if flat: + grown.append(flat) + return tuple(dict.fromkeys(grown)) + + +def _name_matches(relative: str, forms: tuple[str, ...]) -> bool: + name = relative.rsplit("/", 1)[-1] + return any( + fnmatch.fnmatchcase(relative, form) or fnmatch.fnmatchcase(name, form) for form in forms + ) + + +def _glob_files(root: str, arguments: dict[str, Any]) -> str: + pattern = _text(arguments.get("pattern"), required=True, problem=_BAD_PATTERN) + _, start = _place(root, arguments.get("path")) + if not os.path.isdir(start): + raise _Refused(_NOT_A_FOLDER if os.path.exists(start) else _MISSING) + forms = _pattern_forms(pattern) + budget = _Budget() + found: list[str] = [] + size = 0 + capped = False + for path, relative in _walk(root, start, budget): + if not _name_matches(os.path.relpath(path, start), forms): + continue + if size + len(relative) + 1 > _ROOM: + capped = True + break + found.append(relative) + size += len(relative) + 1 + return _finished_text(found, stopped=budget.stopped or capped, nothing="No file matches.") + + +def _grep_files(root: str, arguments: dict[str, Any]) -> str: + text = _text(arguments.get("text"), required=True, problem=_BAD_TEXT) + wanted = _text(arguments.get("glob"), required=False, problem=_BAD_GLOB) + ignore_case = arguments.get("ignore_case") + if ignore_case is None: + ignore_case = False + if not isinstance(ignore_case, bool): + raise _Refused(_BAD_IGNORE_CASE) + joined, start = _place(root, arguments.get("path")) + budget = _Budget() + if os.path.isdir(start): + targets: Iterator[tuple[str, str]] = _walk(root, start, budget) + else: + _one_file(joined) + targets = iter([(joined, os.path.relpath(start, root))]) + forms = _pattern_forms(wanted) if wanted else () + needle = text.lower() if ignore_case else text + exact = text.encode("utf-8") + hits: list[str] = [] + size = 0 + capped = False + for path, relative in targets: + if forms and not _name_matches(relative, forms): + continue + try: + if budget.read_bytes + os.lstat(path).st_size > MAX_READ_BYTES: + budget.stopped = True + break + data = _read_bytes(path) + except (OSError, _Refused): + continue + budget.read_bytes += len(data) + for number, raw in enumerate(data.splitlines(), 1): + # A line this long is minified or generated, and matching it is what makes a + # search slow. + if len(raw) > MAX_LINE_BYTES: + continue + if not ignore_case and exact not in raw: + continue + line = raw.decode("utf-8", "replace") + if ignore_case and needle not in line.lower(): + continue + hit = f"{relative}:{number}:{line[:HIT_CHARS]}" + if size + len(hit) + 1 > _ROOM: + capped = True + break + hits.append(hit) + size += len(hit) + 1 + if capped: + break + return _finished_text(hits, stopped=budget.stopped or capped, nothing="No line matches.") + + +# -- the answer -------------------------------------------------------------------------------- + + +def _stripped(path: str, prefixes: tuple[str, ...]) -> str: + for prefix in prefixes: + if path == prefix.rstrip("/") and prefix != "./": + return "" + if path.startswith(prefix): + return path[len(prefix) :] + return path + + +def _checked_pair( + root: str, prefixes: tuple[str, ...], item: Any +) -> tuple[str, str] | None: + if not isinstance(item, dict): + return None + folder, entry = item.get("folder"), item.get("entry") + if not isinstance(folder, str) or not isinstance(entry, str): + return None + folder, entry = _stripped(folder, prefixes), _stripped(entry, prefixes) + if not fits_path_rule(entry): + return None + if folder and (not fits_path_rule(folder) or not entry.startswith(folder + "/")): + return None + if _in_skipped_folder(entry.split("/")[:-1]): + return None + path = os.path.join(root, entry) + try: + if not stat.S_ISREG(os.lstat(path).st_mode): + return None + except OSError: + return None + real_folder = os.path.realpath(os.path.join(root, folder)) if folder else root + if not _under(real_folder, root) or not os.path.isdir(real_folder): + return None + if not _under(os.path.realpath(path), real_folder): + return None + return folder, entry + + +def _uncarried_entry(root: str, prefixes: tuple[str, ...], item: Any) -> str | None: + """A real file the model named, inside the source, whose path breaks only the platform's + rule (a Next.js ``app/(chat)/...`` route, a space). A missing or outside file is the model's + mistake and isn't one.""" + if not isinstance(item, dict) or not isinstance(item.get("entry"), str): + return None + entry = _stripped(item["entry"], prefixes) + if not entry or len(entry) > MAX_PATH_CHARS or fits_path_rule(entry): + return None + parts = entry.split("/") + if len(parts) > MAX_PATH_PARTS or {"", ".", ".."} & set(parts): + return None + if _in_skipped_folder(parts[:-1]): + return None + path = os.path.join(root, entry) + try: + if not stat.S_ISREG(os.lstat(path).st_mode): + return None + except (OSError, ValueError): + return None + if not _under(os.path.realpath(path), root): + return None + return entry + + +def _checked_pairs( + root: str, prefixes: tuple[str, ...], agents: list[Any] +) -> tuple[tuple[tuple[str, str], ...], int, tuple[str, ...]]: + kept: list[tuple[str, str]] = [] + entries: set[str] = set() + uncarried: list[str] = [] + refused = 0 + for item in agents: + pair = _checked_pair(root, prefixes, item) + if pair is None: + refused += 1 + entry = _uncarried_entry(root, prefixes, item) + if entry is not None: + uncarried.append(entry) + elif pair[1] not in entries: + entries.add(pair[1]) + kept.append(pair) + return tuple(kept), refused, tuple(uncarried) + + +def _submit_server(source: Path, answer: _Answer) -> ToolServer: + root = os.path.realpath(str(source)) + prefixes = tuple( + dict.fromkeys((str(source).rstrip("/") + "/", root.rstrip("/") + "/", "./")) + ) + + async def submit_agents(arguments: dict[str, Any]) -> dict[str, Any]: + try: + if answer.pairs: + return _ok(SUBMIT_AFTER_SUBMIT) + agents = arguments.get("agents") + if not isinstance(agents, list) or len(agents) > MAX_PAIRS: + return _error(_BAD_SUBMIT) + kept, refused, uncarried = await _in_thread( + _checked_pairs, root, prefixes, agents + ) + # Another call may have finished while this one was checking. Nothing awaits + # between this check and the set below, so two calls of one reply, run at once, + # can never both keep their pairs. + if answer.pairs: + return _ok(SUBMIT_AFTER_SUBMIT) + answer.called = True + answer.uncarried.update(uncarried) + if kept: + answer.pairs = kept + answer.refused = refused + return _ok( + f"Recorded {len(kept)}; {refused} refused. " + "Your work is done: end your turn now." + ) + if uncarried: + return _ok( + f"Recorded 0; {refused} refused. {len(set(uncarried))} name a file whose " + "path the platform can't carry: a path may hold only letters, digits, '.', " + "'_', '-' and '/'. Those agents can't be listed, so don't submit them " + "again. Submit any others, or end your turn." + ) + return _ok( + f"Recorded 0; {refused} refused. Each entry must be a file inside the " + "repository and inside its folder. Submit again with corrected paths, or end " + "your turn if there are no agents." + ) + except asyncio.TimeoutError: + return _error(_TOO_SLOW) + except Exception: + logger.warning("find_agents: a tool call failed", exc_info=True) + return _error(_FAILED) + + return ToolServer( + name="scan", + tools=[ + ToolSpec( + name="submit_agents", + description=( + "Submit the agents you found, once: each a folder (\"\" for the root) and " + "the entry file inside it, both relative to the repository's root. An empty " + "list means there are none." + ), + input_schema={ + "type": "object", + "properties": { + "agents": { + "type": "array", + "maxItems": MAX_PAIRS, + "items": { + "type": "object", + "properties": { + "folder": {"type": "string"}, + "entry": {"type": "string"}, + }, + "required": ["folder", "entry"], + }, + } + }, + "required": ["agents"], + }, + handler=submit_agents, + ) + ], + ) diff --git a/src/fi/alk/harness/scan_document.py b/src/fi/alk/harness/scan_document.py new file mode 100644 index 000000000..7c38318b1 --- /dev/null +++ b/src/fi/alk/harness/scan_document.py @@ -0,0 +1,666 @@ +"""The agent scan's document, `agents.json`, in the shape the platform's reader accepts. + +The platform treats a file its reader rejects as a failed reading and retries it, and the retry +would fail the same way. So every row is checked against the reader's rules before it is written, +a row that fails is left out rather than failing the scan, and the whole file is checked once more +before it leaves. The rules mirror the reader's serializer at the commit the fixture names; the +fixture test holds the two together. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +import tempfile +import unicodedata +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from .scan_walk import fits_path_rule + +if TYPE_CHECKING: + from .job import HarnessJob + from .scan_walk import Candidate + +logger = logging.getLogger(__name__) + +AGENTS_FILE = "agents.json" +SCHEMA_VERSION = "futureagi.agent-scan.v1" +MAX_AGENTS = 50 +MAX_BYTES = 1_048_576 +ENV_NAMES_KEPT = 50 +NAME_MAX = 255 +EVIDENCE_MAX = 200 +EVIDENCE_KEPT = 5 + +# The reader's words, in its order. The first ten are the frameworks it names. +FRAMEWORK_NAMES = ( + "langchain", + "langgraph", + "livekit", + "claude_agent_sdk", + "openai_agents", + "pydantic_ai", + "google_adk", + "crewai", + "pipecat", + "deepagents", + "other", + "none", + "unknown", +) +_NAMED = frozenset(FRAMEWORK_NAMES[:10]) +_STATUSES = ("found", "none", "error") +_CONFIDENCES = ("high", "medium", "low") +_CONFIDENCE_RANK = {word: rank for rank, word in enumerate(_CONFIDENCES)} + +# The reader's patterns, as written there. It trims a value before matching, and a trimmed value +# has no trailing line break, so `fullmatch` here gives the same verdicts as its `search`. +_SUBFOLDER_PATTERN = re.compile( + r"^(?!/)(?!.*(^|/)\.{1,2}(/|$))[A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*$" +) +_ENV_NAME_PATTERN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_COMMIT_PATTERN = re.compile(r"^[0-9a-f]{40}$") +_CODE_PATTERN = re.compile(r"^[a-z0-9_]+$") + +_TOP_KEYS = ("schema_version", "status", "commit_sha", "agents", "truncated", "error") +_AGENT_KEYS = ( + "name", + "subfolder", + "entry", + "framework", + "confidence", + "evidence", + "env_names", + "estimates", +) +_FRAMEWORK_KEYS = ("name", "version", "label") +_ESTIMATE_KEYS = ("tools", "prompt_files") +_ERROR_KEYS = ("code", "message") + +# Removed from names: control, format, surrogate and private-use characters. None of them shows +# as text, and a surrogate can't be written as UTF-8 at all. +_HIDDEN_CATEGORIES = frozenset({"Cc", "Cf", "Cs", "Co"}) + +# A sentence's one free part is a path, shortened to the room its template leaves. Every +# template leaves at least this much, so a path always shows something more than its file name. +_PATH_ROOM = 120 +_TEMPLATES = { + "builds": "Builds with {detail} in {where}.", + "graph": "Its langgraph.json lists a graph: {where}.", + "agent_py": "An agent.py using a framework or a model provider: {where}.", + # The provider's name follows "for", so no article has to agree with it. + "client": "Creates a client for {detail} in {where}.", + "model": "Calls a chat model through {detail} in {where}.", + "project": "Its project file is {where}.", + "openai_agents_set_aside": ( + "Uses the OpenAI Agents SDK, set aside because a folder named agents/ shadows it." + ), + "shares_folder": ( + "Another agent here shares this folder; for now only the folder can be picked." + ), + "graphs_may_stop": ( + "This folder's langgraph.json lists more than one graph, so a build may stop." + ), + "agent_py_may_stop": "More than one agent.py sits under this folder, so a build may stop.", + "not_finished": "The framework check for this folder did not finish.", + "partial": "Not every file under this folder could be read, so counts may be short.", + "found_by_ai": "Found by the AI step, not by the code check.", +} +_LONGEST_TEMPLATE = max(len(template) for template in _TEMPLATES.values()) +_UNNAMED_PATH = "a file in this folder" +_DETAIL = re.compile(r"[A-Za-z0-9_.-]{1,40}") +_DETAIL_FALLBACK = { + "builds": "an agent framework", + "client": "a model provider", + "model": "an agent framework", +} +_FRAMEWORK_TITLES = { + "langchain": "LangChain", + "langgraph": "LangGraph", + "livekit": "LiveKit Agents", + "claude_agent_sdk": "the Claude Agent SDK", + "openai_agents": "the OpenAI Agents SDK", + "pydantic_ai": "Pydantic AI", + "google_adk": "Google ADK", + "crewai": "CrewAI", + "pipecat": "Pipecat", + "deepagents": "Deep Agents", +} +# Line numbers past this are a broken fact, and would eat the room the path needs. +_LINE_SHOWN_BELOW = 10_000_000 + + +@dataclass(frozen=True) +class Row: + candidate: Candidate + framework: dict[str, Any] + finished: bool + found_by_ai: bool + + +class ScanResultInvalid(Exception): + """The document as a whole breaks the platform's rules, so writing it would only be retried.""" + + def __init__(self, problem: str) -> None: + super().__init__( + f"The agent list breaks the platform's rules at {problem}; " + "this is a fault in the scan, not in the repository." + ) + self.problem = problem + + +def _clean_name(text: str) -> str: + """``text`` without hidden characters, its runs of any kind of space made one space.""" + # Spaces first: a tab or line break is itself a control character, and dropping it before + # splitting would join the words on either side of it. + words = ( + "".join(ch for ch in word if unicodedata.category(ch) not in _HIDDEN_CATEGORIES) + for word in text.split() + ) + return " ".join(word for word in words if word) + + +def root_name(job: HarnessJob) -> str: + """The name for an agent at the repository's root.""" + choices: list[str] = [] + named = job.metadata.get("agent_name") + if isinstance(named, str): + choices.append(named) + repository = job.source.repository + if isinstance(repository, str): + choices.append(repository.rstrip("/").rsplit("/", 1)[-1].removesuffix(".git")) + for choice in choices: + cleaned = _clean_name(choice) + if cleaned: + return cleaned + return "agent" + + +def _base_name(subfolder: str, root: str) -> str: + last = subfolder.rsplit("/", 1)[-1] if subfolder else "" + for choice in (last, root): + cleaned = _clean_name(choice) + if cleaned: + return cleaned + return "agent" + + +def _names(rows: Sequence[Row], root: str) -> list[str]: + bases = [_base_name(row.candidate.subfolder, root)[:NAME_MAX] for row in rows] + names = list(bases) + for suffix_of in (lambda entry: entry.rsplit("/", 1)[-1], lambda entry: entry): + counts = Counter(names) + names = [ + _with_suffix(base, f" ({suffix_of(row.candidate.entry)})") + if counts[name] > 1 + else name + for base, name, row in zip(bases, names, rows) + ] + return names + + +def _with_suffix(base: str, suffix: str) -> str: + # The suffix is what tells two agents apart, so the base gives way to it, and a suffix too + # long on its own keeps its end, where the entry's file name is. + if len(suffix) >= NAME_MAX: + return suffix[-NAME_MAX:] + return base[: NAME_MAX - len(suffix)] + suffix + + +def _shown_path(path: Any, room: int) -> str: + if not isinstance(path, str) or not fits_path_rule(path): + return _UNNAMED_PATH + if len(path) <= room: + return path + parts = path.split("/") + for start in range(1, len(parts)): + tail = "…/" + "/".join(parts[start:]) + if len(tail) <= room: + return tail + return parts[-1][: max(room - 1, 0)] + "…" + + +def _detail(kind: str, detail: str) -> str: + if kind in ("builds", "model") and detail in _FRAMEWORK_TITLES: + return _FRAMEWORK_TITLES[detail] + if _DETAIL.fullmatch(detail): + return detail + return _DETAIL_FALLBACK.get(kind, "") + + +def _sentence(kind: str, path: Any = "", line: Any = 0) -> str | None: + """The fixed sentence for one fact, or ``None`` for a kind with no sentence. + + A path appears only when the platform's path rule accepts it, so no file name that rule + refuses, and no character that can't be written, ever reaches the document. + """ + head, _, detail = kind.partition(" ") + template = _TEMPLATES.get(head) + if template is None: + return None + words = {"detail": _detail(head, detail), "where": ""} + if "{where}" in template: + at_line = ( + f", line {line}" + if isinstance(line, int) and not isinstance(line, bool) and 0 < line < _LINE_SHOWN_BELOW + else "" + ) + fixed = len(template.format(**words)) + len(at_line) + words["where"] = _shown_path(path, EVIDENCE_MAX - fixed) + at_line + sentence = template.format(**words) + return sentence if len(sentence) <= EVIDENCE_MAX else sentence[: EVIDENCE_MAX - 1] + "…" + + +def _framework_name(row: Row) -> str: + name = row.framework.get("name") if isinstance(row.framework, Mapping) else None + return name if isinstance(name, str) else "" + + +def _confidence(row: Row, shares: bool) -> str: + candidate = row.candidate + name = _framework_name(row) + if ( + name in ("unknown", "none") + or row.found_by_ai + or shares + or candidate.build_may_stop + or candidate.partial + or not row.finished + # A chat model alone may be one step of something larger, not an agent. + or candidate.sign == "model" + ): + return "low" + if name in _NAMED and name in candidate.builds: + return "high" + return "medium" + + +def _order_key(row: Row, shares: bool) -> tuple[bool, int, str, str]: + candidate = row.candidate + return ( + candidate.side, + _CONFIDENCE_RANK[_confidence(row, shares)], + candidate.subfolder, + candidate.entry, + ) + + +def _choice_key(row: Row, models_first: bool) -> tuple[bool, bool, int, str, str]: + side, rank, subfolder, entry = _order_key(row, False) + behind = ( + models_first + and not side + and not row.found_by_ai + and row.candidate.sign in ("client", "model") + ) + return (side, behind, rank, subfolder, entry) + + +def _evidence(row: Row, shares: bool) -> list[str]: + candidate = row.candidate + set_aside_shown = _framework_name(row) != "openai_agents" + facts = [ + _sentence(kind, path, line) + for kind, path, line in candidate.evidence + if kind != "openai_agents_set_aside" or set_aside_shown + ] + caveats = [ + _sentence(kind) + for kind, applies in ( + ("found_by_ai", row.found_by_ai), + ("shares_folder", shares), + ("graphs_may_stop", candidate.build_may_stop == "graphs"), + ("agent_py_may_stop", candidate.build_may_stop == "agent_py"), + ("not_finished", not row.finished), + ("partial", candidate.partial), + ) + if applies + ] + # The strongest fact leads; the caveats that explain a low confidence come before the rest. + known = [sentence for sentence in facts if sentence] + ordered = known[:1] + [sentence for sentence in caveats if sentence] + known[1:] + return list(dict.fromkeys(ordered))[:EVIDENCE_KEPT] + + +def _wire_row(row: Row, *, name: str, shares: bool) -> dict[str, Any]: + candidate = row.candidate + framework = row.framework + return { + "name": name, + "subfolder": candidate.subfolder, + "entry": candidate.entry, + "framework": dict(framework) if isinstance(framework, Mapping) else framework, + "confidence": _confidence(row, shares), + "evidence": _evidence(row, shares), + "env_names": list(candidate.env_names[:ENV_NAMES_KEPT]), + "estimates": {"tools": candidate.tools, "prompt_files": candidate.prompt_files}, + } + + +def _document( + *, + status: str, + commit_sha: str | None, + agents: list[dict[str, Any]], + truncated: bool, + error: dict[str, Any] | None, +) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "status": status, + "commit_sha": commit_sha, + "agents": agents, + "truncated": truncated, + "error": error, + } + + +def _commit(commit_sha: str | None) -> str | None: + if not isinstance(commit_sha, str): + return None + lowered = commit_sha.lower() + return lowered if _COMMIT_PATTERN.fullmatch(lowered) else None + + +def choose_rows( + rows: Sequence[Row], *, limit: int = MAX_AGENTS, models_first: bool = False +) -> tuple[list[Row], bool]: + """The rows to write, one per entry and at most ``limit``, and whether any were left out. + + Who shares a folder isn't known until the rows are chosen, so the order here treats every + row as alone in its folder. With ``models_first``, the model's rows outside side folders + are chosen before the walk's rows that only call a model: the model was paid to look past + those, so what it found is the likelier agent, and it must not be the row the cap drops. + """ + seen: set[str] = set() + unique: list[Row] = [] + for row in rows: + if row.candidate.entry in seen: + continue + seen.add(row.candidate.entry) + unique.append(row) + unique.sort(key=lambda row: _choice_key(row, models_first)) + chosen = unique[: max(limit, 0)] + return chosen, len(chosen) != len(rows) + + +def finish( + *, + rows: Sequence[Row], + root_name: str, + commit_sha: str | None, + truncated: bool, + error: tuple[str, str] | None, + models_first: bool = False, +) -> tuple[bytes, bool]: + """The bytes of ``agents.json``, and whether a row was left out by the cap or the self-check. + + Rows given with an error win: what was found is worth more than why the scan stopped early, + and ``truncated`` says it stopped. ``models_first`` changes which rows are chosen, never + the order they are written in. + """ + root = _clean_name(root_name) or "agent" + commit = _commit(commit_sha) + chosen, dropped = choose_rows(rows, limit=MAX_AGENTS, models_first=models_first) + + kept: list[Row] = [] + for row in chosen: + name = _base_name(row.candidate.subfolder, root)[:NAME_MAX] + alone = _wire_row(row, name=name, shares=False) + problems = wire_problems( + _document(status="found", commit_sha=None, agents=[alone], truncated=False, error=None) + ) + if problems: + logger.warning("agent scan: row left out, problem=%s", problems[0]) + dropped = True + continue + kept.append(row) + + folders = Counter(row.candidate.subfolder for row in kept) + shares = [folders[row.candidate.subfolder] > 1 for row in kept] + order = sorted(range(len(kept)), key=lambda index: _order_key(kept[index], shares[index])) + kept = [kept[index] for index in order] + shares = [shares[index] for index in order] + names = _names(kept, root) + agents = [ + _wire_row(row, name=name, shares=shared) for row, name, shared in zip(kept, names, shares) + ] + + if rows and error is not None: + logger.info("agent scan: error dropped for rows, code=%s", error[0]) + if agents: + status, wire_error = "found", None + elif error is not None and not rows: + status, wire_error = "error", {"code": error[0], "message": error[1]} + else: + status, wire_error = "none", None + document = _document( + status=status, + commit_sha=commit, + agents=agents, + truncated=bool(truncated or dropped or (rows and error is not None)), + error=wire_error, + ) + problems = wire_problems(document) + if problems: + raise ScanResultInvalid(problems[0]) + body = json.dumps(document, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + if len(body) > MAX_BYTES: + raise ScanResultInvalid("document.too_large") + logger.info("agent scan: rows written=%d left out=%d", len(agents), len(rows) - len(agents)) + return body, dropped + + +def _at(where: str, key: str) -> str: + return f"{where}.{key}" if where else key + + +def _keys_problems(value: Any, where: str, keys: tuple[str, ...]) -> list[str]: + if not isinstance(value, dict): + return [_at(where, "not_an_object")] + problems = [_at(_at(where, key), "missing") for key in keys if key not in value] + if set(value) - set(keys): + problems.append(_at(where, "unknown_keys")) + return problems + + +def _text_problems( + value: Any, + where: str, + *, + longest: int, + nullable: bool = False, + blank: bool = False, + pattern: re.Pattern[str] | None = None, +) -> list[str]: + if value is None: + return [] if nullable else [f"{where}.null"] + if not isinstance(value, str): + return [f"{where}.type"] + # The reader trims before every check, so lengths and patterns are judged on the trimmed text. + text = value.strip() + if not text: + return [] if blank else [f"{where}.blank"] + problems = [] + if len(text) > longest: + problems.append(f"{where}.too_long") + if "\x00" in text: + problems.append(f"{where}.null_byte") + if any(0xD800 <= ord(ch) <= 0xDFFF for ch in text): + problems.append(f"{where}.surrogate") + if pattern is not None and not pattern.fullmatch(text): + problems.append(f"{where}.pattern") + return problems + + +def _choice_problems(value: Any, where: str, choices: tuple[str, ...]) -> list[str]: + if value is None: + return [f"{where}.null"] + if not isinstance(value, str) or value not in choices: + return [f"{where}.choice"] + return [] + + +def _count_problems(value: Any, where: str) -> list[str]: + if not isinstance(value, int) or isinstance(value, bool): + return [f"{where}.type"] + return [f"{where}.negative"] if value < 0 else [] + + +def _texts_problems( + value: Any, where: str, *, most: int, longest: int, pattern: re.Pattern[str] | None = None +) -> list[str]: + if not isinstance(value, list): + return [f"{where}.not_a_list"] + problems = [f"{where}.too_many"] if len(value) > most else [] + for index, item in enumerate(value): + problems += _text_problems(item, f"{where}[{index}]", longest=longest, pattern=pattern) + return problems + + +def _framework_problems(value: Any, where: str) -> list[str]: + problems = _keys_problems(value, where, _FRAMEWORK_KEYS) + if not isinstance(value, dict): + return problems + if "name" in value: + problems += _choice_problems(value["name"], f"{where}.name", FRAMEWORK_NAMES) + if "version" in value: + problems += _text_problems(value["version"], f"{where}.version", longest=64, nullable=True) + if "label" in value: + problems += _text_problems(value["label"], f"{where}.label", longest=128, nullable=True) + if not problems and (value["name"] == "other") != (value["label"] is not None): + problems.append(f"{where}.label.other_only") + return problems + + +def _agent_problems(agent: Any, where: str) -> list[str]: + problems = _keys_problems(agent, where, _AGENT_KEYS) + if not isinstance(agent, dict): + return problems + if "name" in agent: + problems += _text_problems(agent["name"], f"{where}.name", longest=NAME_MAX) + for key in ("subfolder", "entry"): + if key in agent: + problems += _text_problems( + agent[key], + f"{where}.{key}", + longest=512, + blank=key == "subfolder", + pattern=_SUBFOLDER_PATTERN, + ) + if "framework" in agent: + problems += _framework_problems(agent["framework"], f"{where}.framework") + if "confidence" in agent: + problems += _choice_problems(agent["confidence"], f"{where}.confidence", _CONFIDENCES) + if "evidence" in agent: + problems += _texts_problems( + agent["evidence"], f"{where}.evidence", most=EVIDENCE_KEPT, longest=EVIDENCE_MAX + ) + if "env_names" in agent: + problems += _texts_problems( + agent["env_names"], + f"{where}.env_names", + most=200, + longest=128, + pattern=_ENV_NAME_PATTERN, + ) + if "estimates" in agent: + estimates = agent["estimates"] + problems += _keys_problems(estimates, f"{where}.estimates", _ESTIMATE_KEYS) + if isinstance(estimates, dict): + for key in _ESTIMATE_KEYS: + if key in estimates: + problems += _count_problems(estimates[key], f"{where}.estimates.{key}") + return problems + + +def _error_problems(error: Any) -> list[str]: + if error is None: + return [] + problems = _keys_problems(error, "error", _ERROR_KEYS) + if not isinstance(error, dict): + return problems + if "code" in error: + problems += _text_problems(error["code"], "error.code", longest=64, pattern=_CODE_PATTERN) + if "message" in error: + problems += _text_problems(error["message"], "error.message", longest=2000) + return problems + + +def wire_problems(document: dict[str, Any]) -> list[str]: + """Where ``document`` breaks the platform reader's rules, as problem names, never values. + + An empty list means the reader accepts it. Where the reader is looser than the written + contract (it ignores unknown keys, and turns numbers into text and ``"true"`` into a boolean), + this is stricter: the kit never needs that leniency, and relying on it would tie the file to + the reader's version. + """ + problems = _keys_problems(document, "", _TOP_KEYS) + if not isinstance(document, dict): + return problems + if "schema_version" in document: + problems += _choice_problems( + document["schema_version"], "schema_version", (SCHEMA_VERSION,) + ) + if "status" in document: + problems += _choice_problems(document["status"], "status", _STATUSES) + if "commit_sha" in document: + problems += _text_problems( + document["commit_sha"], + "commit_sha", + longest=40, + nullable=True, + pattern=_COMMIT_PATTERN, + ) + agents = document.get("agents") + if "agents" in document: + if not isinstance(agents, list): + problems.append("agents.not_a_list") + elif len(agents) > MAX_AGENTS: + problems.append("agents.too_many") + else: + for index, agent in enumerate(agents): + problems += _agent_problems(agent, f"agents[{index}]") + if "truncated" in document and not isinstance(document["truncated"], bool): + problems.append("truncated.type") + if "error" in document: + problems += _error_problems(document["error"]) + if not problems: + status, error = document["status"], document["error"] + if status == "found" and (not agents or error is not None): + problems.append("status.found_needs_agents_and_no_error") + elif status == "none" and (agents or error is not None): + problems.append("status.none_needs_no_agents_and_no_error") + elif status == "error" and (agents or error is None): + problems.append("status.error_needs_an_error_and_no_agents") + return problems + + +def write_agents_json(output: Path, body: bytes) -> Path: + """Write ``body`` as ``output/agents.json`` whole, so a reader never sees half a file.""" + output = Path(output) + output.mkdir(parents=True, exist_ok=True) + path = output / AGENTS_FILE + temporary = tempfile.NamedTemporaryFile( + "wb", dir=output, prefix=".agents-", suffix=".tmp", delete=False + ) + try: + with temporary: + temporary.write(body) + temporary.flush() + os.fsync(temporary.fileno()) + os.chmod(temporary.name, 0o644) + os.replace(temporary.name, path) + except BaseException: + try: + os.unlink(temporary.name) + except OSError: + pass + raise + return path diff --git a/src/fi/alk/harness/scan_folders.py b/src/fi/alk/harness/scan_folders.py new file mode 100644 index 000000000..3fb460e7f --- /dev/null +++ b/src/fi/alk/harness/scan_folders.py @@ -0,0 +1,223 @@ +"""Asks TH-8805's framework check about one agent folder at a time, in a child the scan can stop. + +`detect_framework` bounds its work, not its time, so one slow folder could hold the scan past +its clock. In a child process a check can be abandoned at the deadline by killing it, and a +child the system kills costs that folder's word rather than the whole scan. +""" + +from __future__ import annotations + +import logging +import traceback +import multiprocessing +import os +import time +from collections.abc import Callable +from dataclasses import dataclass +from multiprocessing.connection import Connection +from multiprocessing.process import BaseProcess +from pathlib import Path +from typing import Any + +from .framework_detection import detect_framework +from .reading_entry import MaskSecrets +from .source_folder import SubfolderNotFound, agent_folder + +logger = logging.getLogger(__name__) +logger.addFilter(MaskSecrets()) + +# Loaded once in the fork server, so a restarted child doesn't import the kit again. +_PRELOAD = ["fi.alk.harness.framework_detection"] +# A killed child is gone almost at once; the bound only stops a wedged reap holding the scan. +_JOIN_SECONDS = 5.0 +# The key of a child's answer that carries a failed check's trace instead of a framework. +_FAILED = "failed" + + +@dataclass(frozen=True) +class FolderWord: + framework: dict[str, Any] + finished: bool + + +class FolderCheckerBroken(Exception): + """The child couldn't start, or ended without an answer other than by a signal.""" + + +class ClockRanOut(Exception): + """The scan's deadline passed before a folder's answer came back.""" + + +def _unknown() -> dict[str, Any]: + return {"name": "unknown", "version": None, "label": None} + + +def _answer(source: Path, subfolder: str) -> dict[str, Any]: + try: + # The build's own call for a named folder. Nothing is installed during a scan, so + # there is no package folder to read. + return detect_framework( + agent_folder(source, subfolder) if subfolder else source, + repo_root=source, + package_dir=None, + raise_faults=True, + ) + except SubfolderNotFound: + # A folder the build would refuse has no word; it is not a reason to end the child. + return {"name": "unknown", "version": None} + except Exception: + # A fault in the check itself. The child stays up for the next folder, and sends the + # trace so the parent logs it masked and marks the row unfinished. + return {_FAILED: traceback.format_exc()} + + +def _serve(conn: Connection) -> None: + """The child's loop: one answer per request, until the parent's end closes.""" + while True: + try: + source, subfolder = conn.recv() + conn.send(_answer(Path(source), subfolder)) + except (EOFError, OSError): + return + + +def _framework(answer: Any) -> dict[str, Any]: + """The answer's framework, in the platform's shape; anything malformed is ``unknown``.""" + name = answer.get("name") if isinstance(answer, dict) else None + if not isinstance(name, str): + return _unknown() + label = answer.get("label") if name == "other" else None + return { + "name": name, + "version": None, + "label": label if isinstance(label, str) else None, + } + + +class FolderChecker: + """One child process, started on the first check, answering for one folder at a time. + + Answers are kept per folder for the life of the checker, since agents that share a folder + get the same word and a folder whose check killed one child would likely kill the next. + """ + + pid: int | None + + def __init__(self) -> None: + self.pid = None + self._process: BaseProcess | None = None + self._conn: Connection | None = None + self._words: dict[tuple[str, str], FolderWord] = {} + + def __enter__(self) -> FolderChecker: + return self + + def __exit__(self, *exc: object) -> None: + self._end_child() + + def check( + self, + source: Path, + subfolder: str, + *, + deadline: float, + clock: Callable[[], float] = time.monotonic, + ) -> FolderWord: + if deadline - clock() <= 0: + raise ClockRanOut("The scan's clock ran out before this folder was checked.") + key = (os.fspath(source), subfolder) + known = self._words.get(key) + if known is not None: + return FolderWord(dict(known.framework), known.finished) + if self._process is None: + self._start() + conn = self._conn + assert conn is not None + try: + conn.send((source, subfolder)) + except OSError: + return self._keep(key, self._ended()) + # A deadline that passed while the child started still ends it: a late answer isn't taken. + if not conn.poll(max(deadline - clock(), 0.0)): + self._end_child() + logger.warning("folder checker: the clock ran out, child ended") + raise ClockRanOut("The scan's clock ran out while a folder was being checked.") + try: + answer = conn.recv() + except (EOFError, OSError): + return self._keep(key, self._ended()) + if isinstance(answer, dict) and _FAILED in answer: + logger.warning("folder checker: the check failed in the child\n%s", answer[_FAILED]) + return self._keep(key, FolderWord(_unknown(), finished=False)) + return self._keep(key, FolderWord(_framework(answer), finished=True)) + + def _keep(self, key: tuple[str, str], word: FolderWord) -> FolderWord: + self._words[key] = word + return FolderWord(dict(word.framework), word.finished) + + def _start(self) -> None: + parent: Connection | None = None + child: Connection | None = None + try: + context = multiprocessing.get_context("forkserver") + context.set_forkserver_preload(_PRELOAD) + parent, child = context.Pipe() + process = context.Process(target=_serve, args=(child,), daemon=True) + process.start() + except Exception as error: + if parent is not None: + parent.close() + logger.warning("folder checker: the child could not start", exc_info=True) + raise FolderCheckerBroken( + "The folder checker's child process could not start." + ) from error + finally: + # The child has its own copy. Closing ours lets a dead child read as end of file. + if child is not None: + child.close() + self._process, self._conn, self.pid = process, parent, process.pid + + def _ended(self) -> FolderWord: + """The child stopped answering: a signal costs this folder's word, anything else is broken.""" + process = self._process + assert process is not None + code: int | None = None + try: + # The exit code is only known once the child is reaped. + process.join(_JOIN_SECONDS) + code = process.exitcode + except (OSError, ValueError, AssertionError): + # multiprocessing raises these for a child that is already closed or never started. + logger.warning("folder checker: the child's exit code could not be read", exc_info=True) + code = None + self._end_child() + if code is not None and code < 0: + logger.warning("folder checker: child ended by signal %d", -code) + return FolderWord(_unknown(), finished=False) + logger.warning("folder checker: child ended without an answer, exit code %s", code) + if code is None: + raise FolderCheckerBroken( + "The folder checker's child process stopped answering and was ended." + ) + raise FolderCheckerBroken( + "The folder checker's child process ended without an answer, " + f"with exit code {code}." + ) + + def _end_child(self) -> None: + """Kills, reaps and forgets the child, if there is one. Never raises.""" + process, conn = self._process, self._conn + self._process = self._conn = None + self.pid = None + if process is not None: + try: + process.kill() + process.join(_JOIN_SECONDS) + process.close() + except Exception: + logger.warning("folder checker: the child could not be ended", exc_info=True) + if conn is not None: + try: + conn.close() + except OSError: + logger.debug("folder checker: the pipe was already closed", exc_info=True) diff --git a/src/fi/alk/harness/scan_walk.py b/src/fi/alk/harness/scan_walk.py new file mode 100644 index 000000000..3077c7b61 --- /dev/null +++ b/src/fi/alk/harness/scan_walk.py @@ -0,0 +1,1468 @@ +"""Finds the files in a repository that look like agents, from its files alone. + +One walk reads each file once and runs nothing. Python files go through the framework +check's own per-file reader, so "builds an agent" means what it means to a build, and a +token pass of the scan's own adds what that reader doesn't keep: the repository files a +file imports, provider clients, setting names and tool registrations. + +A candidate imported by another, directly or through files that aren't candidates, is +part of the importer's agent when the importer's sign is as strong or stronger, unless +the importer sits in a side folder and the imported file doesn't. Candidates that import +each other are one agent. +""" + +from __future__ import annotations + +import ast +import codecs +import json +import logging +import math +import os +import posixpath +import re +import time +import tokenize +import warnings +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Iterator, Literal + +from .framework_detection import ( + _ALWAYS_SKIPPED, + _TOP_ONLY_SKIPS, + _FileScan, + _has_long_line, + _is_side_file, + _is_side_part, + _OverBudget, + _own_modules, + _read_file, + _scan_file, + _visible_tokens, + _Work, +) +from .bundle_author_v2 import _IGNORED_ARTIFACT_PARTS as _BUILD_IGNORED +from .reading_entry import MaskSecrets +from .source_folder import _FOLDER_FORM, _FOLDER_MAX + +logger = logging.getLogger(__name__) +logger.addFilter(MaskSecrets()) + +Sign = Literal["framework", "graph", "agent_py", "client", "model"] +Hazard = Literal["", "graphs", "agent_py"] + +_FILE_BYTES = 1_048_576 +# A hundredth of a folder check's allowance: the crafted files that use a whole one +# take seconds, so one file now costs about a second at most. +_FILE_WORK = 1_000_000_000 +# What the walk holds grows with the names each file imports, not its bytes: about 33 KB a +# file for files that each import 200 names (measured 10 Nov, 1,389 MB peak at 40,000 such +# files), which leaves room for the folder check's child in a 4,096 MB sandbox. +_MAX_FILES = 40_000 +_MAX_BYTES = 268_435_456 +_NAMES_PER_FILE = 200 +_IMPORT_DEPTH = 10 +_MAX_AGENTS = 50 +_DESCRIBE_FILES = 2_000 +_DESCRIBE_BYTES = 33_554_432 +_ENV_NAME_MAX = 128 +_MAX_PARTS = 64 +# The prompt count is an estimate, and every file walked is listed, read or not: past this +# many a tree of prompt files costs memory and adds nothing a person would act on. +_MAX_PROMPT_PATHS = 20_000 +# How many folder entries one look for a build's hazard visits. The hazard is advice, so a +# tree too big to look through gives none rather than the walk's time. +_HAZARD_ENTRIES = 200_000 +# No file system the scan runs on has a path this long, so no file can match. +_MODULE_PATH_MAX = 4_096 + +_STRENGTH = {"framework": 3, "graph": 2, "agent_py": 1, "client": 0, "model": 0} +# A file that only talks to a model: listed unless a stronger agent shares its project. +_WEAKEST = frozenset({"client", "model"}) +_PROJECT_FILES = ("pyproject.toml", "requirements.txt", "setup.py") +_PACKAGE_FILE = "package.json" +_TEMPLATES = (".env.example", ".env.sample", ".env.template") +_LANGGRAPH = "langgraph.json" +_SCRIPT_SUFFIXES = (".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs") +_SOURCE_SUFFIXES = (".py", ".ipynb", *_SCRIPT_SUFFIXES) +# Script files nobody writes an agent in: build output, vendored copies, type stubs and tool +# settings. A Python helper beside only these is no reason to pay for the model. +_BUILT_FOLDERS = frozenset({"dist", "build", "static", "vendor", "node_modules"}) +_BUILT_ENDINGS = (".min.js", ".d.ts", ".d.mts", ".d.cts") +_CONFIG_ENDINGS = tuple(".config" + suffix for suffix in _SCRIPT_SUFFIXES) + +_CLIENTS = { + ("OpenAI",): "openai", + ("AsyncOpenAI",): "openai", + ("AzureOpenAI",): "openai", + ("Anthropic",): "anthropic", + ("AsyncAnthropic",): "anthropic", + ("genai", "Client"): "google_genai", + ("Mistral",): "mistral", + ("cohere", "Client"): "cohere", + ("cohere", "ClientV2"): "cohere", + ("Groq",): "groq", + ("litellm", "completion"): "litellm", + ("litellm", "acompletion"): "litellm", +} +_PROVIDER_MODULES = ( + ("openai",), + ("anthropic",), + ("google", "genai"), + ("mistralai",), + ("cohere",), + ("groq",), + ("litellm",), +) +_TOOL_DECORATORS = frozenset({"tool", "function_tool"}) +_SETTING_CALLS = (["os", "getenv"], ["os", "environ", "get"]) +_STRING_PREFIXES = frozenset({"", "r", "u"}) + +_ENV_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +# Only the name is captured: the rest of the line holds a value and is never kept. A second +# `=` straight after the first is base64 padding, the last line of a pasted key, not a setting. +_TEMPLATE_LINE = re.compile(r"[ \t]*(?:export[ \t]+)?([^\s=]+)[ \t]*=(?!=)") +_ENCODED_LENGTH = 16 +# An unquoted value this long in base64's alphabets is a pasted key, which may wrap onto the +# lines below it. +_ENCODED_VALUE = re.compile(r"[A-Za-z0-9+/_-]{40,}={0,2}") +_KEY_BEGIN = "-----BEGIN" +_KEY_END = "-----END" +_DOUBLE_QUOTE_CLOSE = re.compile(r'(? bool: + """Whether the platform can carry a non-empty path. The root, ``""``, isn't one.""" + return ( + bool(path) + and len(path) <= _FOLDER_MAX + and _FOLDER_FORM.fullmatch(path) is not None + ) + + +def _good_env_name(name: str) -> bool: + return len(name) <= _ENV_NAME_MAX and _ENV_NAME.fullmatch(name) is not None + + +def _join(*parts: str) -> str: + return "/".join(part for part in parts if part) + + +def _under(path: str, folder: str) -> bool: + return not folder or path.startswith(folder + "/") + + +def _below(path: str, folder: str) -> str: + return path[len(folder) + 1 :] if folder else path + + +def _is_side_path(path: str) -> bool: + parts = path.split("/") + return any(_is_side_part(part) for part in parts[:-1]) or _is_side_file(parts[-1]) + + +def _is_written_script(path: str) -> bool: + """A TypeScript or JavaScript file someone wrote, so one an agent may be written in.""" + parts = path.split("/") + name = parts[-1] + return ( + name.endswith(_SCRIPT_SUFFIXES) + and not any(part in _BUILT_FOLDERS for part in parts[:-1]) + and not name.endswith(_BUILT_ENDINGS) + and not name.endswith(_CONFIG_ENDINGS) + ) + + +def _has_link(top: str, folder: str) -> bool: + """Whether reaching ``folder`` passes through a link, or it isn't a real folder.""" + current = top + for part in folder.split("/") if folder else (): + current = os.path.join(current, part) + if os.path.islink(current): + return True + return not os.path.isdir(current) + + +def _folders_to_enter(directory: str, names: list[str], *, at_top: bool) -> list[str]: + return [ + name + for name in names + if name not in _ALWAYS_SKIPPED + and not (at_top and name in _TOP_ONLY_SKIPS) + and not os.path.islink(os.path.join(directory, name)) + ] + + +def _chain( + token: tokenize.TokenInfo | None, tokens: Iterator[tokenize.TokenInfo] +) -> tuple[list[str], tokenize.TokenInfo | None]: + """Reads `n(.n)*` from `token` on; returns its parts and the token after it.""" + parts: list[str] = [] + while token is not None and token.type == tokenize.NAME: + if len(parts) < _MAX_PARTS: + parts.append(token.string) + token = next(tokens, None) + if token is None or token.string != ".": + break + token = next(tokens, None) + return parts, token + + +def _string_value(text: str) -> str | None: + """The value of one plain string literal; None for an f-string, bytes, or anything + too long to be a setting's name.""" + if len(text) > _ENV_NAME_MAX + 8: + return None + body = text.lstrip("rRuUbBfFtT") + if text[: len(text) - len(body)].lower() not in _STRING_PREFIXES: + return None + try: + value = ast.literal_eval(text) + except (ValueError, SyntaxError): + return None + return value if isinstance(value, str) else None + + +class _OwnReader: + """The scan's pass over one file's tokens, for what the framework reader drops.""" + + def __init__(self) -> None: + self.imports: dict[tuple[int, tuple[str, ...]], None] = {} + self.provider = False + self.clients: dict[str, int] = {} + self.env: set[str] = set() + self.tools = 0 + + def read(self, tokens: Iterator[tokenize.TokenInfo]) -> None: + prev = "" + at_start = True + token = next(tokens, None) + while token is not None: + text = token.string + if token.type == tokenize.NAME and prev != ".": + if text == "import": + token, prev, at_start = self._plain_import(tokens), "", False + continue + if text == "from": + token, prev, at_start = self._from_import(tokens), "", False + continue + if prev not in ("def", "class"): + line = token.start[0] + parts, token = _chain(token, tokens) + token = self._call(parts, token, tokens, line) + prev, at_start = parts[-1], False + continue + if text == "@" and at_start: + parts, token = _chain(next(tokens, None), tokens) + if parts and parts[-1] in _TOOL_DECORATORS: + self.tools = min(self.tools + 1, _NAMES_PER_FILE) + prev, at_start = (parts[-1] if parts else "@"), False + continue + if token.type == tokenize.NEWLINE: + prev, at_start = "", True + elif token.type not in (tokenize.INDENT, tokenize.DEDENT): + prev, at_start = text, text == ";" + token = next(tokens, None) + + def _call( + self, + parts: list[str], + token: tokenize.TokenInfo | None, + tokens: Iterator[tokenize.TokenInfo], + line: int, + ) -> tokenize.TokenInfo | None: + following = token.string if token is not None else "" + if following == "(": + sdk = _CLIENTS.get(tuple(parts[-2:])) or _CLIENTS.get(tuple(parts[-1:])) + if sdk is not None and len(self.clients) < _NAMES_PER_FILE: + self.clients.setdefault(sdk, line) + if parts in _SETTING_CALLS: + return self._setting(tokens, (",", ")")) + elif following == "[" and parts == ["os", "environ"]: + return self._setting(tokens, ("]",)) + return token + + def _setting( + self, tokens: Iterator[tokenize.TokenInfo], closers: tuple[str, ...] + ) -> tokenize.TokenInfo | None: + # Called just past the bracket. Only a lone literal is a name: what follows a + # comma is a default, which may be a real value, so it is never looked at. + literal = next(tokens, None) + if literal is None or literal.type != tokenize.STRING: + return literal + after = next(tokens, None) + if after is not None and after.string in closers: + name = _string_value(literal.string) + if ( + name is not None + and _good_env_name(name) + and len(self.env) < _NAMES_PER_FILE + ): + self.env.add(name) + return after + + def _plain_import( + self, tokens: Iterator[tokenize.TokenInfo] + ) -> tokenize.TokenInfo | None: + while True: + parts, token = _chain(next(tokens, None), tokens) + if parts: + self._import(0, parts) + if token is not None and token.string == "as": + next(tokens, None) + token = next(tokens, None) + if token is None or token.string != ",": + return token + + def _from_import( + self, tokens: Iterator[tokenize.TokenInfo] + ) -> tokenize.TokenInfo | None: + level = 0 + token = next(tokens, None) + while token is not None and token.string in (".", "..."): + level += len(token.string) + token = next(tokens, None) + module: list[str] = [] + # `import` is a NAME token too, and `from . import x` has no module before it. + if token is not None and token.string != "import": + module, token = _chain(token, tokens) + # `yield from` and `raise ... from` have no `import` after them. + if token is None or token.string != "import" or not (module or level): + return token + token = next(tokens, None) + if token is not None and token.string == "(": + token = next(tokens, None) + while token is not None: + if token.string == "*": + token = next(tokens, None) + elif token.type == tokenize.NAME: + # The name may be a module of its own, or something the module defines. + self._import(level, [*module, token.string]) + token = next(tokens, None) + if token is not None and token.string == "as": + next(tokens, None) + token = next(tokens, None) + else: + break + if token is None or token.string != ",": + break + token = next(tokens, None) + self._import(level, module) + return token + + def _import(self, level: int, parts: list[str]) -> None: + key = (level, tuple(parts[:_MAX_PARTS])) + if level == 0 and any( + key[1][: len(module)] == module for module in _PROVIDER_MODULES + ): + self.provider = True + if ( + len(self.imports) < _NAMES_PER_FILE + and sum(len(part) + 1 for part in key[1]) <= _MODULE_PATH_MAX + ): + self.imports[key] = None + + +def _python_facts(data: bytes, shadowed: frozenset[str]) -> _Facts: + """Raises for a file that has to count as unread.""" + if _has_long_line(data): + raise _Unread() + work = _Work(_FILE_WORK) + scan = _FileScan(shadowed, frozenset(), work) + if _scan_file(data, scan, read_uses=True): + raise _Unread() + own = _OwnReader() + with warnings.catch_warnings(): + # On 3.12 `tokenize` and `literal_eval` warn once per `\{`, which floods stderr. + warnings.simplefilter("ignore") + own.read(_visible_tokens(data, work)) + builds = sorted( + (framework, line) + for (framework, strength), line in scan.found.items() + if strength == "builds" + ) + models = sorted( + (framework, line) + for (framework, strength), line in scan.found.items() + if strength == "model" + ) + openai_agents = min( + ( + line + for (framework, _), line in scan.found.items() + if framework == "openai_agents" + ), + default=0, + ) + return _Facts( + builds=tuple(builds), + models=tuple(models), + found_any=bool(scan.found), + openai_agents=openai_agents, + provider=own.provider, + clients=tuple( + sorted(own.clients.items(), key=lambda item: (item[1], item[0])) + ), + env=tuple(sorted(own.env)), + tools=own.tools, + imports=tuple(own.imports), + ) + + +def _closes(text: str, quote: str) -> bool: + if quote == '"': + # `\\` is a backslash, not an escape, so a quote right after one still closes. + return _DOUBLE_QUOTE_CLOSE.search(text.replace("\\\\", "")) is not None + if quote == "`": + # Node's `.env` reader holds a backtick value open over lines, and closes it as it + # closes a double-quoted one. + return _BACKTICK_CLOSE.search(text.replace("\\\\", "")) is not None + return quote in text + + +def _looks_encoded(name: str) -> bool: + # A key wrapped over lines with no quotes and no block markers can leave a line looking + # like `=`. Setting names are words: short, or one case, or split by `_`. Encoded + # text has a long run mixing two of lower case, upper case and digits; base64url puts `_` + # inside it, and base32 has no lower case, so each run between `_` is judged alone. + for part in name.split("_"): + kinds = ( + any(char.islower() for char in part) + + any(char.isupper() for char in part) + + any(char.isdigit() for char in part) + ) + if len(part) >= _ENCODED_LENGTH and kinds >= 2: + return True + return False + + +def _continues(value: str) -> bool: + # An odd run of backslashes at the end: the last one joins the next line to the value, + # as a shell reads it. + return (len(value) - len(value.rstrip("\\"))) % 2 == 1 + + +def _template_text(data: bytes) -> str: + # `utf-8-sig`, so a byte-order mark isn't read as part of the first name. Names are ASCII, + # so a template saved in another encoding still gives them: UTF-16 says so with its + # byte-order mark, and Latin-1 reads any bytes at all. + try: + return data.decode("utf-8-sig") + except UnicodeDecodeError: + if data.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)): + return data.decode("utf-16") + return data.decode("latin-1") + + +def _template_names(data: bytes) -> set[str]: + """The setting names in a settings template. + + A line inside a value that spans lines is that value's text and never a name: a quoted + value left open, a pasted key's block, a line joined by a trailing backslash, or a key + wrapped over bare lines. A value never closed takes the rest of the file: names lost are + better than a value kept. So is a name that looks like encoded text. + """ + names: set[str] = set() + open_quote = "" + in_key = False + continued = False + # Whether the line above may wrap onto this one: a bare line, which is no setting, or a + # setting whose unquoted value is a long run of encoded text. + wraps = False + # Only `\n` ends a line: no `.env` reader splits on the other breaks `splitlines` knows, + # such as U+2028 or a form feed. + for line in _template_text(data).split("\n"): + line = line.rstrip("\r") + if continued: + continued = _continues(line) + wraps = True + continue + if open_quote or in_key: + if in_key and _KEY_END in line: + in_key = False + if open_quote and _closes(line, open_quote): + open_quote = "" + continue + if not line.strip() or line.lstrip().startswith("#"): + wraps = False + continue + # A comment may mention a key's marker; only one before it opens a block. + begin = line.split(" #", 1)[0].find(_KEY_BEGIN) + if begin >= 0 and _KEY_END not in line[begin:]: + in_key = True + match = _TEMPLATE_LINE.match(line) + if match is None: + wraps = True + continue + value = line[match.end() :].lstrip(" \t") + follows_wrap = wraps + # A pasted key's first line can carry trailing blanks or a note after it. + wraps = _ENCODED_VALUE.fullmatch(value.split(" #", 1)[0].rstrip(" \t")) is not None + quoted = value[:1] in ("'", '"', "`") + if quoted and not _closes(value[1:], value[0]): + open_quote = value[0] + continued = not quoted and _continues(value) + if follows_wrap and not value.strip(): + # `=` straight below a wrapped value is that value's last line, its padding + # read as a setting with no value. + continue + name = match.group(1) + if _good_env_name(name) and not _looks_encoded(name) and len(names) < _NAMES_PER_FILE: + names.add(name) + return names + + +def _script_names(data: bytes) -> set[str]: + names: set[str] = set() + for match in _PROCESS_ENV.finditer(data.decode("utf-8")): + name = next(group for group in match.groups() if group is not None) + if _good_env_name(name) and len(names) < _NAMES_PER_FILE: + names.add(name) + return names + + +class _Projects: + """Which folders hold a project file, looked up once per folder.""" + + def __init__(self, top: str, names: tuple[str, ...]) -> None: + self._top = top + self._names = names + self._file: dict[str, str | None] = {} + + def file_in(self, folder: str) -> str | None: + if folder not in self._file: + found = None + for name in self._names: + path = os.path.join(self._top, folder, name) + if os.path.isfile(path) and not os.path.islink(path): + found = name + break + self._file[folder] = found + return self._file[folder] + + def nearest(self, folder: str) -> str | None: + while True: + if self.file_in(folder) is not None: + return folder + if not folder: + return None + folder = posixpath.dirname(folder) + + +def _sign(path: str, facts: _Facts, graph_files: dict[str, str]) -> Sign | None: + if facts.builds: + return "framework" + if path in graph_files: + return "graph" + if posixpath.basename(path) == "agent.py" and (facts.found_any or facts.provider): + return "agent_py" + if facts.models: + return "model" + if facts.clients: + return "client" + return None + + +def _evidence( + entry: str, + facts: _Facts | None, + *, + graph: str | None, + agent_py: bool, + project: tuple[str, str] | None, + model: bool = False, +) -> tuple[tuple[str, str, int], ...]: + found: list[tuple[str, str, int]] = [] + if facts is not None: + found += [(f"builds {name}", entry, line) for name, line in facts.builds] + if graph is not None: + found.append(("graph", graph, 0)) + if agent_py: + found.append(("agent_py", entry, 0)) + if model and facts is not None: + found += [(f"model {name}", entry, line) for name, line in facts.models] + if facts is not None: + found += [(f"client {sdk}", entry, line) for sdk, line in facts.clients] + if project is not None: + found.append((f"project {project[1]}", _join(project[0], project[1]), 0)) + if facts is not None and facts.openai_agents: + found.append(("openai_agents_set_aside", entry, facts.openai_agents)) + return tuple(found) + + +def _components(nodes: list[str], edges: dict[str, list[str]]) -> list[list[str]]: + """Strongly connected components, by Tarjan's method without recursion.""" + index: dict[str, int] = {} + low: dict[str, int] = {} + stack: list[str] = [] + on_stack: set[str] = set() + components: list[list[str]] = [] + for root in nodes: + if root in index: + continue + work = [(root, 0)] + while work: + node, position = work.pop() + if position == 0: + index[node] = low[node] = len(index) + stack.append(node) + on_stack.add(node) + children = edges.get(node, []) + descended = False + for at in range(position, len(children)): + child = children[at] + if child not in index: + work.append((node, at + 1)) + work.append((child, 0)) + descended = True + break + if child in on_stack: + low[node] = min(low[node], index[child]) + if descended: + continue + if low[node] == index[node]: + component = [] + while True: + member = stack.pop() + on_stack.discard(member) + component.append(member) + if member == node: + break + components.append(component) + if work: + parent = work[-1][0] + low[parent] = min(low[parent], low[node]) + return components + + +def _source_files( + top: str, start: str, *, side: bool | None = None +) -> Iterator[tuple[str, str, str]]: + """Every file below ``start`` the scan may read, as (directory, folder relative to + ``top``, name), in sorted order so that a walk stopped by a limit stops at the same file. + Installed trees and the folders a build skips are never entered. ``side`` False gives only + files outside test, example and other side folders; True gives only the files in them.""" + for directory, directories, files in os.walk(start, followlinks=False): + directories.sort() + files.sort() + relative = directory[len(top) :].split(os.sep)[1:] + if relative and any(name.endswith(".dist-info") for name in directories): + # An installed tree, not the customer's code. + directories[:] = [] + continue + directories[:] = _folders_to_enter(directory, directories, at_top=not relative) + if side is False: + directories[:] = [name for name in directories if not _is_side_part(name)] + folder = "/".join(relative) + for name in files: + if side is None or _read_with_side_files(_join(folder, name)) is side: + yield directory, folder, name + + +def _read_with_side_files(path: str) -> bool: + """Which pass reads ``path``. A settings template (``.env.example``) starts with a dot, which + makes it a side file by name; it belongs to its folder's pass, so a big tests folder can't + keep it from being read.""" + parts = path.split("/") + if parts[-1] in _TEMPLATES: + return any(_is_side_part(part) for part in parts[:-1]) + return _is_side_path(path) + + +class _ShadowedNames: + """Each project's own module names, read once per project, so an import of the + customer's own ``agents`` package isn't read as a framework.""" + + def __init__(self, top: str, projects: _Projects) -> None: + self._top = top + self._projects = projects + self._by_project: dict[str, frozenset[str]] = {} + + def for_folder(self, folder: str) -> frozenset[str]: + project = self._projects.nearest(folder) or "" + if project not in self._by_project: + self._by_project[project] = _own_modules( + Path(self._top, project) if project else Path(self._top), set() + ) + return self._by_project[project] + + +class _Walk: + """One walk of the source. Holds every file's facts until the agents are decided.""" + + def __init__( + self, + source: Path, + *, + deadline: float, + clock: Callable[[], float], + hazards: dict[str, Hazard] | None = None, + ) -> None: + self.source = Path(source) + self.top = os.fspath(source) + self.deadline = deadline + self.clock = clock + self.hazards: dict[str, Hazard] = {} if hazards is None else hazards + self.facts: dict[str, _Facts] = {} + self.graphs: dict[str, str] = {} + self.templates: dict[str, set[str]] = {} + self.prompt_paths: list[str] = [] + self.unread = 0 + # Unread files whose reader raised a kind not foreseen (a kit fault, not the file's). + self.faults = 0 + # When the step after the walk must give up: see _RESULT_GRACE. + self.result_deadline = deadline + _RESULT_GRACE + self.files_read = 0 + self.bytes_read = 0 + self.stopped: Literal["", "limit", "clock", "memory"] = "" + self.source_outside_side = False + self.script_outside_side = False + self.projects = _Projects(self.top, _PROJECT_FILES) + self.shadowed = _ShadowedNames(self.top, self.projects) + self._roots: dict[str, list[str]] = {} + self._edges: dict[str, tuple[str, ...]] = {} + + def run(self) -> None: + # The customer's own code first, then side folders, so tests and examples can't use + # up the file cap before the agent is reached. + for side in (False, True): + for directory, folder, name in _source_files(self.top, self.top, side=side): + if self.clock() > self.deadline: + self.stopped = "clock" + return + self._take(directory, folder, name) + if self.stopped: + return + + def _take(self, directory: str, folder: str, name: str) -> None: + path = _join(folder, name) + full = os.path.join(directory, name) + if name.endswith(_SOURCE_SUFFIXES) and not _is_side_path(path): + self.source_outside_side = True + if _is_written_script(path) and not _is_side_path(path): + self.script_outside_side = True + if ( + "prompt" in path.lower() + and len(self.prompt_paths) < _MAX_PROMPT_PATHS + and not os.path.islink(full) + ): + self.prompt_paths.append(path) + if not (name.endswith(".py") or name == _LANGGRAPH or name in _TEMPLATES): + return + # Every read of one file sits in this one `try`: a file the tokenizer, the JSON + # reader or the decoder rejects is that file's problem, never the walk's. + try: + data = self._read(full) + if data is None: + return + if name.endswith(".py"): + self.facts[path] = _python_facts(data, self.shadowed.for_folder(folder)) + elif name == _LANGGRAPH: + self._read_graphs(folder, path, data) + else: + self.templates.setdefault(folder, set()).update(_template_names(data)) + except _FILE_ERRORS: + self.unread += 1 + except MemoryError: + raise + except Exception: + # A reader fault of a kind not foreseen is still this file's problem, never the + # walk's: the file is unread. The first trace is logged (secret values masked); the rest are + # counted, so a fault that hits every file can't flood the log. + if not self.faults: + logger.warning("agent walk: a file's reader failed", exc_info=True) + self.faults += 1 + self.unread += 1 + + def _read(self, path: str) -> bytes | None: + if self.files_read >= _MAX_FILES: + self.stopped = "limit" + return None + data = _read_file(path, _FILE_BYTES + 1) + self.files_read += 1 + self.bytes_read += len(data) + if self.bytes_read > _MAX_BYTES: + self.stopped = "limit" + return None + if len(data) > _FILE_BYTES: + raise _Unread() + return data + + def _read_graphs(self, folder: str, path: str, data: bytes) -> None: + document = json.loads(data.decode("utf-8")) + graphs = document.get("graphs") if isinstance(document, dict) else None + if not isinstance(graphs, dict): + return + for declaration in list(graphs.values())[:_NAMES_PER_FILE]: + if not isinstance(declaration, str) or ":" not in declaration: + continue + file_name = declaration.rsplit(":", 1)[0] + while file_name.startswith("./"): + file_name = file_name[2:] + target = posixpath.normpath(_join(folder, file_name)) + if target not in (".", "..") and not target.startswith(("../", "/")): + self.graphs.setdefault(target, path) + + def _import_roots(self, folder: str) -> list[str]: + """Where an absolute import from ``folder`` is looked for, nearest first.""" + if folder not in self._roots: + roots: list[str] = [] + current = folder + while True: + if not current or self.projects.file_in(current) is not None: + roots += [current, _join(current, "src")] + if not current: + break + current = posixpath.dirname(current) + self._roots[folder] = roots + return self._roots[folder] + + def _module_file(self, base: str, parts: tuple[str, ...]) -> str | None: + stem = _join(base, *parts) + for path in (f"{stem}.py" if parts else "", _join(stem, "__init__.py")): + if path and path in self.facts: + return path + return None + + def _imported(self, path: str) -> tuple[str, ...]: + """The read files ``path`` imports. An import that matches none is ignored.""" + if path not in self._edges: + folder = posixpath.dirname(path) + found: dict[str, None] = {} + for level, parts in self.facts[path].imports: + target = None + if level == 0: + if parts: + for root in self._import_roots(folder): + target = self._module_file(root, parts) + if target is not None: + break + else: + base: str | None = folder + for _ in range(level - 1): + base = posixpath.dirname(base) if base else None + if base is None: + break + if base is not None: + target = self._module_file(base, parts) + if target is not None and target != path: + found[target] = None + self._edges[path] = tuple(found) + return self._edges[path] + + def _in_time(self) -> None: + if self.clock() > self.result_deadline: + raise _ResultOutOfTime() + + def _reached(self, start: str, candidates: dict[str, Sign]) -> list[str]: + """The candidates ``start`` imports, through files that aren't candidates.""" + reached: list[str] = [] + seen = {start} + frontier = [start] + for _ in range(_IMPORT_DEPTH): + following: list[str] = [] + for path in frontier: + self._in_time() + for target in self._imported(path): + if target in seen: + continue + seen.add(target) + if target in candidates: + reached.append(target) + else: + following.append(target) + frontier = following + if not frontier: + break + return sorted(reached) + + def _absorbs(self, importer: str, imported: str, signs: dict[str, Sign]) -> bool: + return _STRENGTH[signs[importer]] >= _STRENGTH[signs[imported]] and not ( + _is_side_path(importer) and not _is_side_path(imported) + ) + + def agents(self, signs: dict[str, Sign]) -> list[_Agent]: + nodes = sorted(signs) + edges: dict[str, list[str]] = {} + held = 0 + for node in nodes: + edges[node] = [ + target for target in self._reached(node, signs) if self._absorbs(node, target, signs) + ] + held += len(edges[node]) + if held > _MAX_EDGES: + raise _ResultTooBig() + component_of: dict[str, int] = {} + components = _components(nodes, edges) + for number, members in enumerate(components): + for member in members: + component_of[member] = number + taken = { + component_of[target] + for node in nodes + for target in edges[node] + if component_of[target] != component_of[node] + } + entries = [ + min(members, key=lambda path: (-_STRENGTH[signs[path]], len(path), path)) + for number, members in enumerate(components) + if number not in taken + ] + stronger = [ + (self._project_of(path), _is_side_path(path)) + for path, sign in signs.items() + if sign not in _WEAKEST + ] + kept = [] + for entry in sorted(entries): + self._in_time() + side = _is_side_path(entry) + if signs[entry] in _WEAKEST: + project = self._project_of(entry) + # A side candidate never decides for a file outside side folders. + if any( + other == project and (side or not other_side) + for other, other_side in stronger + ): + continue + kept.append((entry, side)) + agents = [] + for entry, side in kept: + self._in_time() + agents.append(_Agent(entry, signs[entry], side, self._subfolder(entry, side, kept))) + return agents + + def _project_of(self, path: str) -> str: + return self.projects.nearest(posixpath.dirname(path)) or "" + + def _subfolder(self, entry: str, side: bool, agents: list[tuple[str, bool]]) -> str: + folder = posixpath.dirname(entry) + if side: + return folder + project = self.projects.nearest(folder) + if project is None: + return folder + # Only agents of this same project count: one in a deeper project of its own + # doesn't move this agent's folder. + sharing = sum( + 1 + for other, other_side in agents + if not other_side and self._project_of(other) == project + ) + return project if sharing == 1 else folder + + def candidate( + self, agent: _Agent, listed: list[_Agent], *, upload: bool + ) -> Candidate: + subfolder = "" if upload else agent.subfolder + deeper: list[str] = [] + if not upload: + # An upload lists one agent, so nothing else claims a folder under the root. + deeper = [ + other.subfolder + for other in listed + if other.subfolder != subfolder and _under(other.subfolder, subfolder) + ] + own = [agent.entry] if agent.entry in self.facts else [] + for path in self.facts: + if ( + path != agent.entry + and _under(path, subfolder) + and not any(_under(path, folder) for folder in deeper) + and not _is_side_path(_below(path, subfolder)) + ): + own.append(path) + code = sorted({name for path in own for name in self.facts[path].env}) + templates = self.templates.get(subfolder, set()) | self.templates.get("", set()) + facts = self.facts.get(agent.entry) + project_file = self.projects.file_in(subfolder) + return Candidate( + entry=agent.entry, + subfolder=subfolder, + sign=agent.sign, + # The subfolder is always the entry's folder or one above it, so the + # entry's path holds every part of it. + side=agent.side, + builds=tuple(name for name, _ in facts.builds) if facts else (), + build_may_stop=build_hazard( + self.source, + subfolder, + deadline=self.deadline, + clock=self.clock, + hazards=self.hazards, + ), + partial=False, + env_names=(*code, *sorted(templates - set(code))), + tools=sum(self.facts[path].tools for path in own), + prompt_files=_prompt_count(self.prompt_paths, subfolder), + evidence=_evidence( + agent.entry, + facts, + graph=self.graphs.get(agent.entry), + agent_py=agent.sign == "agent_py", + model=agent.sign == "model", + project=(subfolder, project_file) if project_file else None, + ), + ) + + def result(self, *, upload: bool) -> WalkResult: + signs: dict[str, Sign] = {} + for path, facts in self.facts.items(): + sign = _sign(path, facts, self.graphs) + if sign is not None: + signs[path] = sign + left_out = 0 + agents = [] + for agent in self.agents(signs): + if fits_path_rule(agent.entry) and ( + not agent.subfolder or fits_path_rule(agent.subfolder) + ): + agents.append(agent) + else: + left_out += 1 + agents.sort( + key=lambda a: (a.side, -_STRENGTH[a.sign], a.subfolder, a.entry) + ) + listed = agents[:1] if upload else agents[:_MAX_AGENTS] + outside = {sign for path, sign in signs.items() if not _is_side_path(path)} + rows = [] + for agent in listed: + self._in_time() + rows.append(self.candidate(agent, listed, upload=upload)) + return WalkResult( + agents=tuple(rows), + more_than_fifty=not upload and len(agents) > _MAX_AGENTS, + stopped=self.stopped, + unread_files=self.unread, + source_outside_side=self.source_outside_side, + candidate_outside_side=any(not _is_side_path(path) for path in signs), + script_outside_side=self.script_outside_side, + only_weak_outside_side=bool(outside) and outside <= _WEAKEST, + paths_left_out=left_out, + upload_others=len(agents) - len(listed) if upload else 0, + upload_others_side=sum(agent.side for agent in agents[len(listed) :]) + if upload + else 0, + ) + + +def _prompt_count(paths: list[str], subfolder: str) -> int: + return sum( + 1 + for path in paths + if _under(path, subfolder) and "prompt" in _below(path, subfolder).lower() + ) + + +def _empty_result(walk: _Walk, stopped: Literal["limit", "memory", "clock"]) -> WalkResult: + return WalkResult( + agents=(), + more_than_fifty=False, + stopped=stopped, + unread_files=walk.unread, + source_outside_side=walk.source_outside_side, + candidate_outside_side=False, + script_outside_side=False, + only_weak_outside_side=False, + paths_left_out=0, + upload_others=0, + upload_others_side=0, + ) + + +def walk_repo( + source: Path, + *, + deadline: float, + upload: bool = False, + clock: Callable[[], float] = time.monotonic, + hazards: dict[str, Hazard] | None = None, +) -> WalkResult: + """``hazards`` is one scan's answers from ``build_hazard``, shared with ``describe_entry``.""" + walk = _Walk(source, deadline=deadline, clock=clock, hazards=hazards) + try: + walk.run() + except MemoryError: + # Only finished rows survive a memory error, and the walk has none yet. + walk.facts.clear() + walk.graphs.clear() + walk.templates.clear() + walk.prompt_paths.clear() + logger.warning("agent walk: stopped=memory unread=%d", walk.unread) + return _empty_result(walk, "memory") + # Any other fault here is the kit's, not the repository's: it goes to the scan's crash + # path, never to "the repository is larger than the scan reads". + try: + result = walk.result(upload=upload) + except _ResultOutOfTime: + # Joining a huge import graph into agents ran past its deadline: the scan ends in its + # own file (scan_timed_out), never at the helper's hard stop. + logger.warning("agent walk: joining files into agents ran out of time") + walk.facts.clear() + return _empty_result(walk, "clock") + except _ResultTooBig: + # Too many candidates reach each other to hold: the same end as the walk's other caps. + logger.warning("agent walk: joining files into agents passed its edge budget") + walk.facts.clear() + return _empty_result(walk, "limit") + logger.info( + "agent walk: files_read=%d unread=%d reader_faults=%d agents=%d left_out=%d stopped=%s", + walk.files_read, + result.unread_files, + walk.faults, + len(result.agents), + result.paths_left_out, + result.stopped or "no", + ) + return result + + +def build_hazard( + source: Path, + subfolder: str, + *, + deadline: float = math.inf, + clock: Callable[[], float] = time.monotonic, + hazards: dict[str, Hazard] | None = None, +) -> Hazard: + """Why a build of the folder may stop on its own fallback, or ``""``. + + ``hazards`` holds one scan's answers by folder, so a folder many agents share is looked + through once. The look stops, with ``""``, at the deadline or after ``_HAZARD_ENTRIES`` + entries: it walks folders the scan skips, which the walk's own clock never saw. + """ + if hazards is not None and subfolder in hazards: + return hazards[subfolder] + hazard = _hazard(source, subfolder, deadline=deadline, clock=clock) + if hazards is not None: + hazards[subfolder] = hazard + return hazard + + +def _hazard( + source: Path, subfolder: str, *, deadline: float, clock: Callable[[], float] +) -> Hazard: + top = os.fspath(source) + if _has_link(top, subfolder): + return "" + folder = os.path.join(top, subfolder) if subfolder else top + try: + data = _read_file(os.path.join(folder, _LANGGRAPH), _FILE_BYTES + 1) + document = json.loads(data.decode("utf-8")) + graphs = document.get("graphs") if isinstance(document, dict) else None + if isinstance(graphs, dict) and len(graphs) > 1: + return "graphs" + except (OSError, ValueError, RecursionError): + # No readable langgraph.json, so it declares no second graph; the agent.py count follows. + logger.debug("hazard: langgraph.json not read", exc_info=True) + # The build counts every `agent.py` below, side folders and the scan's skipped + # folders included, ignoring only its own short list. + found = 0 + entries = 0 + # os.walk reports no listing errors unless asked, so nothing here is expected to raise. + for index, (_, directories, files) in enumerate(os.walk(folder, followlinks=False)): + # The folder's own listing is one read and always looked at; the clock bounds the + # folders below it. + if index and clock() > deadline: + return "" + directories[:] = [name for name in directories if name not in _BUILD_IGNORED] + found += files.count("agent.py") + if found > 1: + return "agent_py" + entries += len(directories) + len(files) + if entries > _HAZARD_ENTRIES: + return "" + return "" + + +class _Description: + """The reads behind one AI-found agent, bounded by its own caps and the clock.""" + + def __init__( + self, + top: str, + entry: str, + subfolder: str, + *, + deadline: float, + clock: Callable[[], float], + ) -> None: + self.top = top + self.entry = entry + self.subfolder = subfolder + self.deadline = deadline + self.clock = clock + self.projects = _Projects(top, _PROJECT_FILES) + self.facts: dict[str, _Facts] = {} + self.script_env: set[str] = set() + self.templates: set[str] = set() + self.prompt_files = 0 + self.files_read = 0 + self.bytes_read = 0 + # A cap or the clock: the reading stops here. + self.partial = False + # One file that couldn't be read or parsed: the counts are short, but the reading + # goes on, as the walk goes on past an unread file. + self.short = False + # Reader faults of a kind not foreseen; only the first one's trace is logged. + self.faults = 0 + self.shadowed = _ShadowedNames(top, self.projects) + + def run(self) -> None: + if _has_link(self.top, self.subfolder): + # Nothing under it is read, so its counts are short. + self.partial = True + return + for folder in dict.fromkeys((self.subfolder, "")): + for name in _TEMPLATES: + if self._out_of_time(): + return + data = self._read(os.path.join(self.top, folder, name)) + if self.partial: + return + if data is not None: + try: + self.templates |= _template_names(data) + except _FILE_ERRORS: + # Its names are missing, so the row's settings are short. + self.short = True + except MemoryError: + raise + except Exception: + self._fault("a template's reader failed") + start = os.path.join(self.top, self.subfolder) if self.subfolder else self.top + for directory, folder, name in _source_files(self.top, start): + if self._out_of_time(): + return + self._take(directory, folder, name) + if self.partial: + return + + def _fault(self, what: str) -> None: + # A fault in the kit that hits every file logs one trace, not one per file. + if not self.faults: + logger.warning("agent description: %s", what, exc_info=True) + self.faults += 1 + self.short = True + + def _out_of_time(self) -> bool: + if self.clock() > self.deadline: + self.partial = True + return self.partial + + def _take(self, directory: str, folder: str, name: str) -> None: + path = _join(folder, name) + full = os.path.join(directory, name) + below = _below(path, self.subfolder) + if "prompt" in below.lower() and not os.path.islink(full): + self.prompt_files += 1 + if path != self.entry and _is_side_path(below): + return + if not name.endswith((".py", *_SCRIPT_SUFFIXES)): + return + data = self._read(full) + if data is None: + return + try: + if name.endswith(".py"): + self.facts[path] = _python_facts(data, self.shadowed.for_folder(folder)) + else: + self.script_env |= _script_names(data) + except _FILE_ERRORS: + # Its facts are missing, so the row's counts are short. + self.short = True + except MemoryError: + raise + except Exception: + self._fault("a file's reader failed") + + def _read(self, path: str) -> bytes | None: + """The file's bytes, or None for a file that can't or mustn't be read; sets + ``partial`` when a cap is reached.""" + if self.files_read >= _DESCRIBE_FILES: + self.partial = True + return None + try: + data = _read_file(path, _FILE_BYTES + 1) + except FileNotFoundError: + # A settings template that isn't there is the usual case, not a short count. + return None + except (OSError, ValueError): + # Unreadable, a link, or a name the system can't open: not read, so the counts + # are short. + self.short = True + return None + self.files_read += 1 + self.bytes_read += len(data) + if self.bytes_read > _DESCRIBE_BYTES: + self.partial = True + return None + if len(data) > _FILE_BYTES: + # Over the size a file is read at: not read, so the counts are short. + self.short = True + return None + return data + + +def describe_entry( + source: Path, + entry: str, + folder: str, + *, + deadline: float, + upload: bool = False, + clock: Callable[[], float] = time.monotonic, + hazards: dict[str, Hazard] | None = None, +) -> Candidate | None: + """The row for one file the AI step kept; None only when its path is refused.""" + if not fits_path_rule(entry): + return None + top = os.fspath(source) + entry_folder = posixpath.dirname(entry) + projects = _Projects(top, (*_PROJECT_FILES, _PACKAGE_FILE)) + if upload: + subfolder = "" + elif (not folder or fits_path_rule(folder)) and _under(entry, folder): + subfolder = folder + else: + nearest = projects.nearest(entry_folder) + subfolder = entry_folder if nearest is None else nearest + reading = _Description(top, entry, subfolder, deadline=deadline, clock=clock) + try: + reading.run() + except MemoryError: + raise + except Exception: + # Runs after the model was paid for, so a fault in the kit keeps what was read and + # marks the row partial rather than ending in a paid retry. The trace is logged, with the + # platform's secret values masked. + logger.warning("agent description stopped", exc_info=True) + reading.partial = True + facts = reading.facts.get(entry) + code = {name for found in reading.facts.values() for name in found.env} + code |= reading.script_env + project_file = projects.file_in(subfolder) + partial = reading.partial or reading.short + if not partial: + logger.info("agent description: files_read=%d", reading.files_read) + else: + logger.info("agent description: files_read=%d partial=yes", reading.files_read) + return Candidate( + entry=entry, + subfolder=subfolder, + sign="client", + side=_is_side_path(entry), + builds=tuple(name for name, _ in facts.builds) if facts else (), + build_may_stop=build_hazard( + source, subfolder, deadline=deadline, clock=clock, hazards=hazards + ), + partial=partial, + env_names=(*sorted(code), *sorted(reading.templates - code)), + tools=sum(found.tools for found in reading.facts.values()), + prompt_files=reading.prompt_files, + evidence=_evidence( + entry, + facts, + graph=None, + agent_py=False, + project=(subfolder, project_file) if project_file else None, + ), + ) diff --git a/src/fi/alk/harness/scenario_source.py b/src/fi/alk/harness/scenario_source.py index 053e100d3..4d01f76da 100644 --- a/src/fi/alk/harness/scenario_source.py +++ b/src/fi/alk/harness/scenario_source.py @@ -645,6 +645,27 @@ def load_scenarios( return scenarios +def authored_agent_document( + contract_text: str, *, subfolder: str | None, framework: Any +) -> dict[str, Any]: + """The agent document stored for a saved contract, ``{"sdk_version", "agent"}``. + + Both a build and a reading with no build make it here, so the two can't differ. The caller + reads the contract, so what it checked is what is used. + """ + contract = AgentContract.model_validate(json.loads(contract_text)).model_dump(mode="json") + # The echo proves which folder was built, so it comes from the contract. + built_from = str((contract.get("runtime") or {}).get("source_subfolder") or "") + if built_from != (subfolder or ""): + raise AgentBuildError("agent_invalid", "subfolder_mismatch") + return { + "sdk_version": _sdk_version, + "agent": build_authored_agent( + contract, subfolder=built_from or None, framework=framework + ), + } + + class BundleScenarioSource: """The real `ScenarioSource`: reads and compiles the bundle's own scenario documents. `hosted_entrypoint.run_job` wires this in only when the injected source is still the default @@ -735,22 +756,11 @@ async def build( if source_kind in (SourceKind.GITHUB, SourceKind.ARCHIVE): framework = await self._detect_framework() try: - contract_file = bundle_dir / "contract.json" - contract = AgentContract.model_validate( - json.loads(contract_file.read_text(encoding="utf-8")) - ).model_dump(mode="json") - # The echo proves which folder was built, so it comes from the contract. - built_from = str( - (contract.get("runtime") or {}).get("source_subfolder") or "" + authored_agent = authored_agent_document( + (bundle_dir / "contract.json").read_text(encoding="utf-8"), + subfolder=job.source.subfolder, + framework=framework, ) - if built_from != (job.source.subfolder or ""): - raise AgentBuildError("agent_invalid", "subfolder_mismatch") - authored_agent = { - "sdk_version": _sdk_version, - "agent": build_authored_agent( - contract, subfolder=built_from or None, framework=framework - ), - } except Exception as exc: # A field path can name the customer's own keys, so only the mismatch is named. mismatch = getattr(exc, "detail", "") == "subfolder_mismatch" diff --git a/src/fi/alk/harness/session.py b/src/fi/alk/harness/session.py index 8b1ca2aaa..739f04ba1 100644 --- a/src/fi/alk/harness/session.py +++ b/src/fi/alk/harness/session.py @@ -217,9 +217,12 @@ def __init__( name: str = "", backend: HarnessBackend | None = None, overheard: bool = True, + idle_retries: int | None = None, ) -> None: self._spec = spec self._backend = backend + # None follows the process-wide setting, read when each message is sent. + self._idle_retries = idle_retries # Whether this stage's turns are echoed to the run's conversation channel. self._overheard = overheard self._session: HarnessSession | None = None @@ -454,14 +457,15 @@ async def say( self, message: str, *, on_event: Callable[[Event], None] | None = None ) -> Turn: """Send a message and wait for the whole reply.""" - for attempt in range(STAGE_IDLE_RETRIES + 1): + retries = STAGE_IDLE_RETRIES if self._idle_retries is None else self._idle_retries + for attempt in range(retries + 1): try: async for event in self.stream(message): if on_event: on_event(event) return self.history[-1] except StageIdleTimeout: - if attempt >= STAGE_IDLE_RETRIES: + if attempt >= retries: raise # A timed-out receive has been cancelled and the provider session may still be # waiting on a dead stream. Reusing it can only reproduce the dead stream. diff --git a/src/fi/alk/harness/skills/find-agents/SKILL.md b/src/fi/alk/harness/skills/find-agents/SKILL.md new file mode 100644 index 000000000..2f3bbd763 --- /dev/null +++ b/src/fi/alk/harness/skills/find-agents/SKILL.md @@ -0,0 +1,86 @@ +--- +name: find-agents +description: Reads a repository in which a plain code check found no agent outside test and example folders, apart from any files that only call a model, and names the folder and entry file of each runnable AI agent it holds, including agents in TypeScript or JavaScript, which that check can't see. +--- + +# Find the agents in a repository + +A plain code check has already looked at this repository and found no agent outside test and +example folders, apart from any files that only call a model, which are already listed. It can't +see agents in TypeScript or JavaScript. You are the second look. Your answer is a list of agents, each a folder and the +entry file that runs it. Code checks every pair you give: a path that isn't a file in the +repository is refused, so a guess costs you a turn and adds nothing. + +Nobody is reading along and nobody will answer a question. Work, submit, end your turn. + +## Your tools + +You have exactly three tools for reading, and one for answering: + +- `mcp__scan_files__read`: one file, by `path`, with optional `offset` (first line, from 1) and + `limit` (at most 400 lines). +- `mcp__scan_files__glob`: file names matching `pattern` (for example `*.py` or `**/agent*.ts`), + under an optional `path`. +- `mcp__scan_files__grep`: lines containing `text`, under an optional `path`, with an optional + `glob` for file names and `ignore_case`. `text` is plain text, matched exactly as written. It + is not a regular expression: `OpenAI(` finds `OpenAI(`, and `a|b` finds only `a|b`. +- `mcp__scan__submit_agents`: your answer. + +Any other tool the host shows you (`Read`, `Grep`, `Bash` and the rest) is refused and wastes a +turn. Paths are relative to the repository's root. Some files are hidden on purpose: settings, +keys and credentials. Don't try to reach them. + +A result that ends "Stopped early at the search limit" is incomplete: search a narrower `path` +before you conclude something isn't there. + +## What an agent looks like + +An agent is code that runs a model in a loop or a graph: it sends a model messages, gives it +tools or steps, and acts on what comes back. Its entry is the file you would run, or the file +that builds the agent object something else serves. + +- **Python:** a file that builds an agent with a framework (LangGraph, LangChain, CrewAI, + LlamaIndex, Pydantic AI, the OpenAI Agents SDK, Google ADK, AutoGen, smolagents, LiveKit and + the like), or calls a provider's client (`OpenAI(`, `Anthropic(`, `genai.Client(`, + `litellm.completion(`) in a loop with tools. Often `agent.py`, `main.py`, `app.py`, + `graph.py` or `server.py`, near a `pyproject.toml` or `requirements.txt`. +- **TypeScript and JavaScript:** a file that imports `@langchain/*`, `@openai/agents`, `ai` (the + Vercel AI SDK), `@mastra/*`, `openai` or `@anthropic-ai/sdk` and runs the loop, often + `index.ts`, `agent.ts`, `server.ts` or `src/main.ts`, near a `package.json`. +- **Notebooks:** an `.ipynb` whose cells build and run an agent counts only if nothing else in + its folder does; prefer a `.py` or `.ts` file beside it. + +The folder is the agent's own folder: the one holding its project file, or else the entry's own +folder. Use `""` for the repository's root. + +## What to skip + +Skip tests, examples, samples, demos, docs, notebooks of experiments, benchmarks, vendored and +third-party code, built or generated output (`dist`, `build`, `*.min.js`, generated clients), +and libraries the repository only installs. A helper that calls a model once, with no loop and +no tools, is not an agent. Several entry files that run the same agent are one agent: name the +one you would run. + +## How to work + +Start broad and narrow down. `glob` for project files and likely entry names, `grep` for the +imports and client calls above, then `read` the few files that decide it. Don't read files whose +answer you already have. + +## Answering + +Call `mcp__scan__submit_agents` once, with every agent you found: + +```json +{"agents": [{"folder": "services/support", "entry": "services/support/agent.py"}]} +``` + +- `entry` is a file inside `folder`, both relative to the root. At most 50. +- If there are none, submit an empty list: `{"agents": []}`. +- If the reply says pairs were refused and none kept, fix the paths and submit again. +- Once a submit is recorded, the work is done: end your turn. Further calls change nothing. + +A turn that ends without a submit is the end of the step, and nothing you found is kept. + +Never quote file content in anything you write: not code, not settings, not values. Name files by +their path and nothing more. diff --git a/src/fi/alk/harness/spend.py b/src/fi/alk/harness/spend.py index 34fb36b77..a9ecd6683 100644 --- a/src/fi/alk/harness/spend.py +++ b/src/fi/alk/harness/spend.py @@ -13,6 +13,7 @@ from __future__ import annotations import json +import logging import os import tempfile from pathlib import Path @@ -20,16 +21,23 @@ JOURNAL_ALIAS = "ALK_SPEND_JOURNAL" +logger = logging.getLogger(__name__) + _stages: dict[str, dict[str, Any]] = {} _path: Path | None = None +_complete: bool | None = None + +def journal_to(path: str | os.PathLike[str] | None, *, complete: bool | None = None) -> bool: + """Where to keep the ledger. Nothing is written until this is set or the alias names a path. -def journal_to(path: str | os.PathLike[str] | None) -> None: - """Where to keep the ledger. Nothing is written until this is set or the alias names a path.""" - global _path + ``complete`` is written beside the totals only when given: a reading's bill says whether every + turn's spend is in it, and a build's ledger has no such key. ``True`` when the file was written. + """ + global _path, _complete _path = Path(path) if path else None - if _path is not None: - _flush() + _complete = complete + return _flush() if _path is not None else False def _destination() -> Path | None: @@ -126,10 +134,15 @@ def snapshot() -> dict[str, Any]: } -def _flush() -> None: +def _flush() -> bool: destination = _destination() if destination is None: - return + return False + body = snapshot() + if _complete is not None: + # A turn with no price makes the total a floor, and a floor billed as the total is wrong. + body["complete"] = _complete and body["unpriced_turns"] == 0 + handle = None try: destination.parent.mkdir(parents=True, exist_ok=True) # Written whole, then moved: a poll that reads mid-write must never see half a total. @@ -137,8 +150,16 @@ def _flush() -> None: "w", dir=destination.parent, prefix=".spend-", suffix=".json", delete=False ) with handle as writing: - json.dump(snapshot(), writing, indent=2, sort_keys=True) + json.dump(body, writing, indent=2, sort_keys=True) os.replace(handle.name, destination) except OSError: # Accounting must never be the reason a stage fails. - pass + logger.warning("spend file not written: %s", destination, exc_info=True) + if handle is not None: + # Left behind, it would sit in a folder the platform reads as the run's output. + try: + Path(handle.name).unlink(missing_ok=True) + except OSError: + pass + return False + return True diff --git a/src/fi/alk/harness/understand.py b/src/fi/alk/harness/understand.py index 0d7d76371..dddf12036 100644 --- a/src/fi/alk/harness/understand.py +++ b/src/fi/alk/harness/understand.py @@ -40,6 +40,11 @@ "the contract." ) PROVIDER_IMPORT_PROFILE_PATH_ENV = "ALK_PROVIDER_IMPORT_PROFILE_PATH" +# Said once when a model ends its turn without saving; every unattended reading uses these words. +SUBMIT_REMINDER = ( + "Nothing was saved: you finished without calling submit_contract. Call it now " + "with the contract you worked out." +) def _provider_import_briefing() -> str: @@ -106,6 +111,23 @@ def _eval_catalogue_briefing(available_evals: list[dict[str, Any]] | None) -> st ) +def reading_settings(job: Any, *, interactive: bool = False) -> dict[str, Any]: + """The ``open_stage`` arguments a job's agent is read with, by a build or by a reading alone. + + One choice for both, so a contract a reading saves is one the build's gate would keep. A + generic job read unattended must declare its setup, which takes more turns. + """ + metadata = getattr(job, "metadata", None) if job is not None else None + metadata = metadata if isinstance(metadata, dict) else {} + offered = metadata.get("available_evals") + generic = metadata.get("generic_harness_v1") is True and not interactive + return { + "available_evals": offered if isinstance(offered, list) else None, + "require_setup_contract": generic, + "max_turns": 140 if generic else 70, + } + + def open_stage( source: AgentSource, *, diff --git a/src/fi/alk/providers/agent.py b/src/fi/alk/providers/agent.py index e0cb43529..18cf6dbdd 100644 --- a/src/fi/alk/providers/agent.py +++ b/src/fi/alk/providers/agent.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging from collections.abc import Mapping, Sequence from typing import Any, Literal from urllib.parse import urlsplit, urlunsplit @@ -8,6 +9,8 @@ from .setup_hash import compute_setup_hash +logger = logging.getLogger(__name__) + AGENT_SCHEMA_VERSION = "1.0" MAIN_UNIT_ID = "main" @@ -245,7 +248,13 @@ def _redact(value: Any, key: str = "") -> Any: if isinstance(value, list): return [_redact(item, key) for item in value] if isinstance(value, str) and "://" in value: - parsed = urlsplit(value) + try: + parsed = urlsplit(value) + except ValueError as refused: + # A netloc urlsplit refuses may still hold credentials, so none of it is kept, and + # only the error's class is logged: its text repeats the netloc. + logger.warning("a URL was redacted whole: class=%s", type(refused).__name__) + return _REDACTED if parsed.scheme and parsed.netloc: hostname = parsed.hostname or "" try: diff --git a/tests/harness/fixtures/agent_scan_e07d56fcd.json b/tests/harness/fixtures/agent_scan_e07d56fcd.json new file mode 100644 index 000000000..8be1142ed --- /dev/null +++ b/tests/harness/fixtures/agent_scan_e07d56fcd.json @@ -0,0 +1,625 @@ +{ + "serializer_commit": "e07d56fcd", + "cases": [ + { + "name": "found_one_agent", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "none_no_agents", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"none\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "error_with_error", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\", \"message\": \"The scan ran out of time.\"}}", + "accepted": true + }, + { + "name": "top_level_not_an_object", + "document_text": "[{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}]", + "accepted": false + }, + { + "name": "schema_version_other", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v2\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "schema_version_missing", + "document_text": "{\"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "status_unknown_word", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"partial\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "status_trailing_space", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found \", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "status_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "found_without_agents", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "found_with_error", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\", \"message\": \"The scan ran out of time.\"}}", + "accepted": false + }, + { + "name": "none_with_agents", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"none\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "none_with_error", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"none\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\", \"message\": \"The scan ran out of time.\"}}", + "accepted": false + }, + { + "name": "error_without_error", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "error_with_agents", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\", \"message\": \"The scan ran out of time.\"}}", + "accepted": false + }, + { + "name": "commit_null", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": null, \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "commit_uppercase", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789ABCDEF0123456789ABCDEF01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "commit_39", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef0123456\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "commit_41", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef012345670\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "commit_empty", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "commit_not_hex", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"gggggggggggggggggggggggggggggggggggggggg\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "commit_trailing_line_break", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\\n\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "commit_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "agents_50", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a0.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a1.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a2.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a3.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a4.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a5.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a6.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a7.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a8.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a9.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a10.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a11.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a12.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a13.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a14.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a15.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a16.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a17.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a18.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a19.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a20.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a21.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a22.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a23.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a24.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a25.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a26.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a27.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a28.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a29.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a30.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a31.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a32.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a33.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a34.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a35.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a36.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a37.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a38.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a39.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a40.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a41.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a42.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a43.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a44.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a45.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a46.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a47.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a48.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a49.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "agents_51", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a0.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a1.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a2.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a3.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a4.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a5.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a6.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a7.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a8.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a9.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a10.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a11.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a12.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a13.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a14.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a15.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a16.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a17.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a18.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a19.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a20.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a21.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a22.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a23.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a24.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a25.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a26.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a27.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a28.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a29.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a30.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a31.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a32.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a33.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a34.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a35.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a36.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a37.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a38.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a39.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a40.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a41.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a42.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a43.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a44.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a45.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a46.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a47.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a48.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a49.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a50.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "agents_not_a_list", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": {\"0\": {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}}, \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "agents_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "agent_not_an_object", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [\"support\"], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "truncated_true", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": true, \"error\": null}", + "accepted": true + }, + { + "name": "truncated_null", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": null, \"error\": null}", + "accepted": false + }, + { + "name": "truncated_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"error\": null}", + "accepted": false + }, + { + "name": "error_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false}", + "accepted": false + }, + { + "name": "error_code_64", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\", \"message\": \"The scan ran out of time.\"}}", + "accepted": true + }, + { + "name": "error_code_65", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\", \"message\": \"The scan ran out of time.\"}}", + "accepted": false + }, + { + "name": "error_code_uppercase", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"Scan\", \"message\": \"The scan ran out of time.\"}}", + "accepted": false + }, + { + "name": "error_code_hyphen", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"scan-x\", \"message\": \"The scan ran out of time.\"}}", + "accepted": false + }, + { + "name": "error_code_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"message\": \"The scan ran out of time.\"}}", + "accepted": false + }, + { + "name": "error_message_empty", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\", \"message\": \"\"}}", + "accepted": false + }, + { + "name": "error_message_blank", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\", \"message\": \" \"}}", + "accepted": false + }, + { + "name": "error_message_2000", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\", \"message\": \"mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm\"}}", + "accepted": true + }, + { + "name": "error_message_2001", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\", \"message\": \"mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm\"}}", + "accepted": false + }, + { + "name": "error_message_null_byte", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\", \"message\": \"a\\u0000b\"}}", + "accepted": false + }, + { + "name": "error_message_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\"}}", + "accepted": false + }, + { + "name": "error_not_an_object", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": \"scan_timed_out\"}", + "accepted": false + }, + { + "name": "name_255", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "name_256", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "name_255_astral", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "name_256_astral", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "name_255_and_a_trailing_space", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn \", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "name_empty", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "name_ideographic_spaces_only", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"\\u3000\\u2003\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "name_inner_unicode_spaces", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"a\\u3000b\\u2003c\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "name_null_byte", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"sup\\u0000port\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "name_lone_surrogate", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"sup\\ud800port\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "name_trailing_line_break", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\\n\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "name_null", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": null, \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "name_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_root", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "subfolder_blank", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"\\u3000\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "subfolder_512", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/bc\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "subfolder_513", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/bcd\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_leading_slash", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"/services\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_dot_dot", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/../x\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_dot", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/./x\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_only_dot_dot", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"..\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_three_dots", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"...\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "subfolder_double_slash", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services//x\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_trailing_slash", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_space", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"my services\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_non_ascii", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"servi\\u00e7es\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_trailing_line_break", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services\\n\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "subfolder_null", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": null, \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "entry_512", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/bc\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "entry_513", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/bcd\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "entry_empty", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "entry_backslash", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services\\\\graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "entry_null_byte", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"graph\\u0000.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "entry_lone_surrogate", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"graph\\udcff.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "entry_trailing_line_break", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\\n\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "entry_inner_line_break", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services\\ngraph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "entry_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_label_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_version_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_null", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": null, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_unknown_word", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"autogen\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_each_word", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"langchain.py\", \"framework\": {\"name\": \"langchain\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"langgraph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"livekit.py\", \"framework\": {\"name\": \"livekit\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"claude_agent_sdk.py\", \"framework\": {\"name\": \"claude_agent_sdk\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"openai_agents.py\", \"framework\": {\"name\": \"openai_agents\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"pydantic_ai.py\", \"framework\": {\"name\": \"pydantic_ai\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"google_adk.py\", \"framework\": {\"name\": \"google_adk\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"crewai.py\", \"framework\": {\"name\": \"crewai\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"pipecat.py\", \"framework\": {\"name\": \"pipecat\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"deepagents.py\", \"framework\": {\"name\": \"deepagents\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"other.py\", \"framework\": {\"name\": \"other\", \"version\": null, \"label\": \"x\"}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"none.py\", \"framework\": {\"name\": \"none\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"unknown.py\", \"framework\": {\"name\": \"unknown\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "framework_version_64", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": \"1111111111111111111111111111111111111111111111111111111111111111\", \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "framework_version_65", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": \"11111111111111111111111111111111111111111111111111111111111111111\", \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_version_empty", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": \"\", \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_other_with_label", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"other\", \"version\": null, \"label\": \"llama_index\"}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "framework_other_without_label", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"other\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_other_empty_label", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"other\", \"version\": null, \"label\": \"\"}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_named_with_label", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": \"llama_index\"}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_label_128", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"other\", \"version\": null, \"label\": \"llllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllll\"}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "framework_label_129", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"other\", \"version\": null, \"label\": \"lllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllll\"}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "confidence_low", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"low\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "confidence_medium", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"medium\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "confidence_uppercase", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"HIGH\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "confidence_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "evidence_empty", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "evidence_5", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"e\", \"e\", \"e\", \"e\", \"e\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "evidence_6", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"e\", \"e\", \"e\", \"e\", \"e\", \"e\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "evidence_200", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "evidence_201", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "evidence_blank", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "evidence_null_byte", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"a\\u0000b\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "evidence_lone_surrogate", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"a\\udcffb\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "evidence_null_item", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [null], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "evidence_not_a_list", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": \"Builds with LangGraph.\", \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "env_names_200", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"N0\", \"N1\", \"N2\", \"N3\", \"N4\", \"N5\", \"N6\", \"N7\", \"N8\", \"N9\", \"N10\", \"N11\", \"N12\", \"N13\", \"N14\", \"N15\", \"N16\", \"N17\", \"N18\", \"N19\", \"N20\", \"N21\", \"N22\", \"N23\", \"N24\", \"N25\", \"N26\", \"N27\", \"N28\", \"N29\", \"N30\", \"N31\", \"N32\", \"N33\", \"N34\", \"N35\", \"N36\", \"N37\", \"N38\", \"N39\", \"N40\", \"N41\", \"N42\", \"N43\", \"N44\", \"N45\", \"N46\", \"N47\", \"N48\", \"N49\", \"N50\", \"N51\", \"N52\", \"N53\", \"N54\", \"N55\", \"N56\", \"N57\", \"N58\", \"N59\", \"N60\", \"N61\", \"N62\", \"N63\", \"N64\", \"N65\", \"N66\", \"N67\", \"N68\", \"N69\", \"N70\", \"N71\", \"N72\", \"N73\", \"N74\", \"N75\", \"N76\", \"N77\", \"N78\", \"N79\", \"N80\", \"N81\", \"N82\", \"N83\", \"N84\", \"N85\", \"N86\", \"N87\", \"N88\", \"N89\", \"N90\", \"N91\", \"N92\", \"N93\", \"N94\", \"N95\", \"N96\", \"N97\", \"N98\", \"N99\", \"N100\", \"N101\", \"N102\", \"N103\", \"N104\", \"N105\", \"N106\", \"N107\", \"N108\", \"N109\", \"N110\", \"N111\", \"N112\", \"N113\", \"N114\", \"N115\", \"N116\", \"N117\", \"N118\", \"N119\", \"N120\", \"N121\", \"N122\", \"N123\", \"N124\", \"N125\", \"N126\", \"N127\", \"N128\", \"N129\", \"N130\", \"N131\", \"N132\", \"N133\", \"N134\", \"N135\", \"N136\", \"N137\", \"N138\", \"N139\", \"N140\", \"N141\", \"N142\", \"N143\", \"N144\", \"N145\", \"N146\", \"N147\", \"N148\", \"N149\", \"N150\", \"N151\", \"N152\", \"N153\", \"N154\", \"N155\", \"N156\", \"N157\", \"N158\", \"N159\", \"N160\", \"N161\", \"N162\", \"N163\", \"N164\", \"N165\", \"N166\", \"N167\", \"N168\", \"N169\", \"N170\", \"N171\", \"N172\", \"N173\", \"N174\", \"N175\", \"N176\", \"N177\", \"N178\", \"N179\", \"N180\", \"N181\", \"N182\", \"N183\", \"N184\", \"N185\", \"N186\", \"N187\", \"N188\", \"N189\", \"N190\", \"N191\", \"N192\", \"N193\", \"N194\", \"N195\", \"N196\", \"N197\", \"N198\", \"N199\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "env_names_201", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"N0\", \"N1\", \"N2\", \"N3\", \"N4\", \"N5\", \"N6\", \"N7\", \"N8\", \"N9\", \"N10\", \"N11\", \"N12\", \"N13\", \"N14\", \"N15\", \"N16\", \"N17\", \"N18\", \"N19\", \"N20\", \"N21\", \"N22\", \"N23\", \"N24\", \"N25\", \"N26\", \"N27\", \"N28\", \"N29\", \"N30\", \"N31\", \"N32\", \"N33\", \"N34\", \"N35\", \"N36\", \"N37\", \"N38\", \"N39\", \"N40\", \"N41\", \"N42\", \"N43\", \"N44\", \"N45\", \"N46\", \"N47\", \"N48\", \"N49\", \"N50\", \"N51\", \"N52\", \"N53\", \"N54\", \"N55\", \"N56\", \"N57\", \"N58\", \"N59\", \"N60\", \"N61\", \"N62\", \"N63\", \"N64\", \"N65\", \"N66\", \"N67\", \"N68\", \"N69\", \"N70\", \"N71\", \"N72\", \"N73\", \"N74\", \"N75\", \"N76\", \"N77\", \"N78\", \"N79\", \"N80\", \"N81\", \"N82\", \"N83\", \"N84\", \"N85\", \"N86\", \"N87\", \"N88\", \"N89\", \"N90\", \"N91\", \"N92\", \"N93\", \"N94\", \"N95\", \"N96\", \"N97\", \"N98\", \"N99\", \"N100\", \"N101\", \"N102\", \"N103\", \"N104\", \"N105\", \"N106\", \"N107\", \"N108\", \"N109\", \"N110\", \"N111\", \"N112\", \"N113\", \"N114\", \"N115\", \"N116\", \"N117\", \"N118\", \"N119\", \"N120\", \"N121\", \"N122\", \"N123\", \"N124\", \"N125\", \"N126\", \"N127\", \"N128\", \"N129\", \"N130\", \"N131\", \"N132\", \"N133\", \"N134\", \"N135\", \"N136\", \"N137\", \"N138\", \"N139\", \"N140\", \"N141\", \"N142\", \"N143\", \"N144\", \"N145\", \"N146\", \"N147\", \"N148\", \"N149\", \"N150\", \"N151\", \"N152\", \"N153\", \"N154\", \"N155\", \"N156\", \"N157\", \"N158\", \"N159\", \"N160\", \"N161\", \"N162\", \"N163\", \"N164\", \"N165\", \"N166\", \"N167\", \"N168\", \"N169\", \"N170\", \"N171\", \"N172\", \"N173\", \"N174\", \"N175\", \"N176\", \"N177\", \"N178\", \"N179\", \"N180\", \"N181\", \"N182\", \"N183\", \"N184\", \"N185\", \"N186\", \"N187\", \"N188\", \"N189\", \"N190\", \"N191\", \"N192\", \"N193\", \"N194\", \"N195\", \"N196\", \"N197\", \"N198\", \"N199\", \"N200\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "env_name_128", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "env_name_129", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "env_name_leading_digit", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"1KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "env_name_hyphen", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI-KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "env_name_trailing_line_break", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\\n\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "env_name_lone_surrogate", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"KEY\\ud800\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "env_names_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "estimates_zero", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 0, \"prompt_files\": 0}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "estimates_negative", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": -1, \"prompt_files\": 0}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "estimates_bool", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": true, \"prompt_files\": 0}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "estimates_prompt_files_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 0}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "estimates_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"]}], \"truncated\": false, \"error\": null}", + "accepted": false + } + ] +} diff --git a/tests/harness/test_agent_reading.py b/tests/harness/test_agent_reading.py new file mode 100644 index 000000000..6cb89b154 --- /dev/null +++ b/tests/harness/test_agent_reading.py @@ -0,0 +1,2160 @@ +"""Reading one agent with no build: the files it leaves, what it is charged, and how it stops. + +The model is a fake backend behind ``session.resolve``, so the real stage, the reading's own +stage, the contract server and the browser-agent gate all run; only the provider is scripted. +""" + +from __future__ import annotations + +import argparse +import asyncio +import builtins +import dataclasses +import json +import logging +import os +import shutil +import signal +import subprocess +import sys +import threading +import types +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest + +import fi.alk +from fi.alk.harness import ( + agent_reading, + authoring_entrypoint, + reading_entry, + reading_result, + scenario_source, + session, + spend, +) +from fi.alk.harness.backends import StageDone +from fi.alk.harness.config import chosen_model +from fi.alk.harness.contract import AgentContract +from fi.alk.harness.framework_detection import detect_framework +from fi.alk.harness.job import HarnessJob, SourceKind +from fi.alk.harness.reading_entry import READING_JOB_INVALID, run_reading_mode +from fi.alk.harness.tools import CONTRACT_SERVER, _checked_contract +from fi.alk.harness.understand import SUBMIT_REMINDER, reading_settings +from fi.alk.providers.agent import build_authored_agent + +EXAMPLES = Path(__file__).resolve().parents[2] / "examples" / "frameworks" +SUPPORT = "agents/support" +SCRAPER = "agents/scraper" +MIB = 1_048_576 +CHAT = { + "agent": "support", + "modality": "chat", + "real_use_cases": ["order"], + "tools": [{"name": "add", "args": ["item_id"]}], +} +# As the gate saves it for a job that names the folder. +IN_SUPPORT = { + **CHAT, + "runtime": {"language": "python", "workdir": SUPPORT, "source_subfolder": SUPPORT}, +} +_LIMIT_ENV = "ALK_READING_MAX_DURATION_SECONDS" + + +class _LoudExit: + """``os`` for the helper under test: its hard limit ending this test process is a failure, + never the silent exit 0 its backstop gives a finished reading.""" + + def __getattr__(self, name): + return getattr(os, name) + + @staticmethod + def _exit(_code): + os.write(2, b"a reading's hard limit ended the test process\n") + os._exit(97) + + +@pytest.fixture(autouse=True) +def _fresh_state(monkeypatch, tmp_path): + monkeypatch.setattr(reading_entry, "os", _LoudExit()) + monkeypatch.delenv(spend.JOURNAL_ALIAS, raising=False) + monkeypatch.setenv("ALK_HARNESS_MODEL", "gemini-2.5-flash") + # The machine's own secrets files must not decide what a test sees masked. + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (tmp_path / "no-secrets.json",)) + # A scripted model that never answers must end a test, not hang it. + monkeypatch.setenv(_LIMIT_ENV, "20") + spend._stages.clear() + spend.journal_to(None) + yield + spend._stages.clear() + spend.journal_to(None) + + +# -- the fake provider -------------------------------------------------------------------------- + + +class _Live: + """One session of the fake backend; each message sent plays the next scripted turn.""" + + def __init__(self, backend: "_Backend", spec) -> None: + self.backend = backend + self.spec = spec + self.interrupted = asyncio.Event() + + async def start(self) -> None: + if self.backend.saves_on_start is not None: + _save(self.backend.output, self.backend.saves_on_start) + if self.backend.start_error is not None: + raise self.backend.start_error + if self.backend.start_raises: + raise FileNotFoundError("the Claude CLI was not found") + + async def stop(self) -> None: + if self.backend.close_hangs: + # A session stuck shutting down; only the loop's own teardown ends it. + await asyncio.Event().wait() + output = self.backend.output + self.backend.stopped.append( + { + "failure": (output / "failure.json").exists(), + "agent": (output / "agent.json").exists(), + } + ) + + async def send(self, message: str) -> None: + self.backend.sent.append(message) + + async def interrupt(self) -> bool: + self.backend.interrupts.append(time.monotonic()) + if self.backend.interrupt_raises: + raise self.backend.interrupt_error(self.backend.interrupt_says) + self.interrupted.set() + return True + + def replies(self): + turn = self.backend.turns.pop(0) if self.backend.turns else _ends() + return turn(self) + + +class _Backend: + default_model = "gemini-2.5-flash" + + def __init__(self, name: str, output: Path, turns: list) -> None: + self.name = name + self.output = output + self.turns = turns + self.created: list = [] + self.sent: list[str] = [] + self.interrupts: list[float] = [] + self.stopped: list[dict] = [] + self.seen: list = [] + self.interrupt_raises = False + self.interrupt_says = "the interrupt control request timed out" + self.interrupt_error: type[BaseException] = RuntimeError + self.start_raises = False + self.saves_on_start: dict | None = None + self.close_hangs = False + self.start_error: BaseException | None = None + self.submissions = 0 + + def create(self, spec) -> _Live: + self.created.append(spec) + return _Live(self, spec) + + def can_drive(self, model: str) -> bool: + return True + + +@pytest.fixture +def model(tmp_path, monkeypatch): + def install(*turns, name: str = "claude", output: Path | None = None) -> _Backend: + fake = _Backend(name, output or tmp_path / "reading", list(turns)) + monkeypatch.setattr(session, "resolve", lambda *a, **k: fake) + return fake + + return install + + +def _done(**fields) -> StageDone: + return StageDone(**{"outcome": "success", "turns": 1, "cost_usd": 0.0, **fields}) + + +def _ends(**fields): + async def turn(live): + yield _done(**fields) + + return turn + + +async def _gate(live: _Live, arguments: dict) -> None: + """What the model's call does: the real submit_contract from the reading's own spec.""" + tool = next( + spec for spec in live.spec.servers[CONTRACT_SERVER].tools if spec.name == "submit_contract" + ) + output = live.backend.output + # A first thin submission is sent back once, so the model submits again. + for _ in range(3): + live.backend.submissions += 1 + await tool.handler(dict(arguments)) + if (output / "contract.json").exists() or (output / "failure.json").exists(): + return + + +def _submits(arguments: dict, **fields): + async def turn(live): + await _gate(live, arguments) + yield _done(**fields) + + return turn + + +def _save(output: Path, contract: dict | None = None, *, text: str | None = None) -> None: + output.mkdir(parents=True, exist_ok=True) + (output / "contract.json").write_text( + text if text is not None else json.dumps(contract), encoding="utf-8" + ) + + +def _writes(contract: dict | None = None, *, text: str | None = None, **fields): + """A turn whose contract the gate has already accepted and saved.""" + + async def turn(live): + _save(live.backend.output, contract, text=text) + yield _done(**fields) + + return turn + + +async def _quiet(live): + await asyncio.Event().wait() + yield _done() + + +# -- the job, the source and the run ------------------------------------------------------------ + + +def _job(*, subfolder: str | None = None, source: dict | None = None, metadata=None) -> dict: + job = { + "job_id": "job", + "run_id": "run", + "execution": "hosted", + "source": source + or { + "kind": "github", + "visibility": "public", + "repository": "acme/agent", + "commit_sha": "a" * 40, + **({"subfolder": subfolder} if subfolder else {}), + }, + "runtime": {"isolation": "dedicated_vm"}, + "agent": {"connector": "auto"}, + } + if metadata is not None: + job["metadata"] = metadata + return job + + +def _repo(root: Path) -> Path: + source = root / "source" + (source / SUPPORT).mkdir(parents=True) + (source / SUPPORT / "agent.py").write_text("x = 1\n", encoding="utf-8") + (source / SCRAPER).mkdir(parents=True) + (source / SCRAPER / "agent.py").write_text( + "from playwright.sync_api import sync_playwright\nsync_playwright()\n", encoding="utf-8" + ) + return source + + +def _run( + root: Path, + job: dict | None = None, + *, + source: Path | None = None, + output=None, + wait: float = 1.0, +) -> int: + job_path = root / "job.json" + job_path.write_text(json.dumps(job or _job()), encoding="utf-8") + return run_reading_mode( + agent_reading._read_agent(interrupt_wait=wait), + job_path=job_path, + source=source if source is not None else root / "source", + output=output if output is not None else root / "reading", + ) + + +def _failure_in(output: Path) -> dict: + return json.loads((output / "failure.json").read_text(encoding="utf-8")) + + +def _cost_in(output: Path) -> dict: + return json.loads((output / "cost.json").read_text(encoding="utf-8")) + + +def _names(output: Path) -> set[str]: + return {path.name for path in output.iterdir()} + + +def _assert_cost(output: Path, total: float, complete: bool) -> None: + cost = _cost_in(output) + assert cost["total_usd"] == pytest.approx(total) + assert cost["complete"] is complete + + +def _short_idle(monkeypatch, seconds: float = 0.2) -> None: + real = agent_reading.open_stage + + def opened(*args, **kwargs): + stage, destination = real(*args, **kwargs) + stage.spec.idle_timeout_seconds = seconds + return stage, destination + + monkeypatch.setattr(agent_reading, "open_stage", opened) + + +def _few_turns(monkeypatch, turns: int) -> None: + real = agent_reading.open_stage + + def opened(*args, **kwargs): + stage, destination = real(*args, **kwargs) + stage.spec.max_turns = turns + return stage, destination + + monkeypatch.setattr(agent_reading, "open_stage", opened) + + +def _short_limit(monkeypatch, soft: float = 0.3) -> None: + monkeypatch.setenv(_LIMIT_ENV, str(soft)) + + +# -- R1 to R9: the files a reading leaves ------------------------------------------------------- + + +def test_a_reading_leaves_the_contract_the_document_and_the_cost(tmp_path, model): + source = _repo(tmp_path) + model(_submits(CHAT, cost_usd=0.25)) + output = tmp_path / "reading" + + assert _run(tmp_path, _job(subfolder=SUPPORT)) == 0 + + assert _names(output) == {"contract.json", "agent.json", "cost.json"} + document = json.loads((output / "agent.json").read_text(encoding="utf-8")) + assert set(document) == {"sdk_version", "agent"} + assert document["sdk_version"] == fi.alk.__version__ + agent = document["agent"] + assert agent["extension"]["source"] == {"subfolder": SUPPORT} + contract = AgentContract.model_validate( + json.loads((output / "contract.json").read_text(encoding="utf-8")) + ).model_dump(mode="json") + framework = detect_framework(source / SUPPORT, repo_root=source, package_dir=None) + assert agent == build_authored_agent(contract, subfolder=SUPPORT, framework=framework) + _assert_cost(output, 0.25, True) + + +@pytest.mark.parametrize( + ("write", "expected"), + [ + ( + lambda folder: (folder / "graph.py").write_text( + "from langgraph.graph import StateGraph\nStateGraph(int)\n", encoding="utf-8" + ), + {"name": "langgraph", "version": None}, + ), + ( + lambda folder: shutil.copy(EXAMPLES / "llamaindex_agent.py", folder), + {"name": "other", "version": None, "label": "llama_index"}, + ), + ], + ids=["langgraph", "other"], +) +def test_the_framework_is_a_builds_shape(tmp_path, model, write, expected): + source = _repo(tmp_path) + write(source / SUPPORT) + model(_writes(IN_SUPPORT)) + + assert _run(tmp_path, _job(subfolder=SUPPORT)) == 0 + + agent = json.loads((tmp_path / "reading" / "agent.json").read_text(encoding="utf-8")) + assert agent["agent"]["extension"]["framework"] == expected + + +def test_with_no_folder_named_the_framework_comes_from_the_contracts_folder(tmp_path, model): + source = _repo(tmp_path) + shutil.copy(EXAMPLES / "llamaindex_agent.py", source / SUPPORT) + # A different framework at the root, so reading the root instead would show. + (source / "graph.py").write_text( + "from langgraph.graph import StateGraph\nStateGraph(int)\n", encoding="utf-8" + ) + model(_writes({**CHAT, "runtime": {"language": "python", "workdir": SUPPORT}})) + + assert _run(tmp_path) == 0 + + agent = json.loads((tmp_path / "reading" / "agent.json").read_text(encoding="utf-8")) + assert agent["agent"]["extension"]["framework"] == { + "name": "other", + "version": None, + "label": "llama_index", + } + + +def test_a_browser_agent_is_refused_by_the_real_gate(tmp_path, model): + _repo(tmp_path) + fake = model(_submits({"modality": "browser"}, cost_usd=0.37)) + output = tmp_path / "reading" + + assert _run(tmp_path, _job(subfolder=SCRAPER)) == 78 + + assert _failure_in(output)["code"] == "agent_type_unsupported" + # One second look before any refusal: the browser answer to it stands and is refused. + assert fake.submissions == 2 + assert len(fake.sent) == 1 + assert not (output / "agent.json").exists() + assert not (output / "contract.json").exists() + _assert_cost(output, 0.37, True) + + +def test_a_missing_folder_is_refused_as_a_build_refuses_it(tmp_path, model): + _repo(tmp_path) + fake = model() + output = tmp_path / "reading" + + assert _run(tmp_path, _job(subfolder="agents/missing")) == 78 + + failure = _failure_in(output) + assert (failure["code"], failure["stage"]) == ("subfolder_not_found", "acquiring_source") + assert fake.created == [] + _assert_cost(output, 0.0, True) + + +def _no_source(root: Path) -> Path: + return root / "source" + + +def _a_file(root: Path) -> Path: + (root / "source").write_text("not a folder", encoding="utf-8") + return root / "source" + + +def _an_empty_folder(root: Path) -> Path: + (root / "source").mkdir() + return root / "source" + + +@pytest.mark.parametrize( + ("make", "subfolder"), + [ + (_no_source, None), + (_a_file, None), + (_an_empty_folder, None), + (_no_source, SUPPORT), + ], + ids=["missing", "a_file", "empty", "missing_with_a_folder"], +) +def test_a_missing_or_empty_source_is_retryable_and_costs_nothing( + tmp_path, model, make, subfolder +): + source = make(tmp_path) + fake = model() + output = tmp_path / "reading" + + assert _run(tmp_path, _job(subfolder=subfolder), source=source) == 1 + + failure = _failure_in(output) + assert (failure["code"], failure["domain"], failure["retryable"]) == ( + "reading_source_empty", + "platform_sync", + True, + ) + assert fake.created == [] + _assert_cost(output, 0.0, True) + + +@pytest.mark.parametrize( + "kind", + [SourceKind.REMOTE, SourceKind.PROVIDER], + ids=["remote", "provider"], +) +def test_only_github_or_an_upload_is_read(tmp_path, model, kind): + _repo(tmp_path) + fake = model() + parsed = HarnessJob.model_validate(_job()) + + other = parsed.model_copy( + update={"source": parsed.source.model_copy(update={"kind": kind})} + ) + refused = agent_reading._check_job(other) + assert refused is not None + assert (refused.code, refused.domain.value, refused.retryable) == ( + READING_JOB_INVALID, + "platform_sync", + False, + ) + for accepted in (SourceKind.GITHUB, SourceKind.ARCHIVE): + assert ( + agent_reading._check_job( + parsed.model_copy( + update={"source": parsed.source.model_copy(update={"kind": accepted})} + ) + ) + is None + ) + + if kind is SourceKind.REMOTE: + remote = _job(source={"kind": "remote", "endpoint": "https://agent.example.com/chat"}) + HarnessJob.model_validate(remote) + assert _run(tmp_path, remote) == 78 + assert _failure_in(tmp_path / "reading")["code"] == READING_JOB_INVALID + assert fake.created == [] + + +@pytest.mark.parametrize( + "agent", + [ + {"connector": "vapi", "mode": "provider_import", "config": {"assistant_id": "asst-1"}}, + {"connector": "vapi", "mode": "connect_only", "config": {"assistant_id": "asst-1"}}, + ], + ids=["provider_import", "connect_only"], +) +def test_a_provider_defined_agent_is_not_read_without_a_build(tmp_path, model, agent): + _repo(tmp_path) + fake = model() + output = tmp_path / "reading" + + assert _run(tmp_path, {**_job(), "agent": agent}) == 78 + + failure = _failure_in(output) + assert (failure["code"], failure["domain"], failure["retryable"]) == ( + READING_JOB_INVALID, + "platform_sync", + False, + ) + assert "repository only" in failure["message"] + assert fake.created == [] + + +def test_a_cancel_before_the_save_is_a_crash_with_its_file(tmp_path, model): + _repo(tmp_path) + + async def cancelled(live): + raise asyncio.CancelledError() + yield _done() + + model(cancelled) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_crashed", True) + assert failure["message"].startswith("CancelledError") + _assert_cost(output, 0.0, False) + + +def test_a_cancel_after_the_save_keeps_the_saved_contract(tmp_path, model): + _repo(tmp_path) + + async def saves_then_cancelled(live): + _save(live.backend.output, CHAT) + raise asyncio.CancelledError() + yield _done() + + model(saves_then_cancelled) + output = tmp_path / "reading" + + assert _run(tmp_path) == 0 + + assert _names(output) == {"contract.json", "agent.json", "cost.json"} + _assert_cost(output, 0.0, False) + + +def test_a_session_whose_close_hangs_still_ends_with_its_saved_contract( + tmp_path, model, monkeypatch, caplog +): + _repo(tmp_path) + _short_limit(monkeypatch) + + async def saves_then_goes_quiet(live): + _save(live.backend.output, CHAT) + await asyncio.Event().wait() + yield _done() + + fake = model(saves_then_goes_quiet) + fake.close_hangs = True + output = tmp_path / "reading" + job_path = tmp_path / "job.json" + job_path.write_text(json.dumps(_job()), encoding="utf-8") + # A short hard limit, so a reading stuck on the close ends this test (exit 97), not CI's clock. + mode = dataclasses.replace( + agent_reading._read_agent(interrupt_wait=0.3), limit_margin_seconds=5.0 + ) + + with caplog.at_level("WARNING", logger=agent_reading.__name__): + assert run_reading_mode(mode, job_path=job_path, source=tmp_path / "source", output=output) == 0 + + assert _names(output) == {"contract.json", "agent.json", "cost.json"} + # Decided while the close still hung: it never got as far as recording itself. + assert fake.stopped == [] + assert any("did not close" in record.getMessage() for record in caplog.records) + + +def test_a_cancel_during_the_stop_is_still_a_timeout(tmp_path, model, monkeypatch): + _repo(tmp_path) + _short_limit(monkeypatch) + fake = model(_quiet) + fake.interrupt_raises = True + fake.interrupt_error = asyncio.CancelledError + output = tmp_path / "reading" + + assert _run(tmp_path, wait=0.3) == 78 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_timed_out", False) + + +def test_an_invalid_saved_contract_logs_no_values_from_it(tmp_path, model, caplog): + _repo(tmp_path) + marker = "CUSTOMER-MARKER-7f3a db_password=hunter2" + model(_writes(text=json.dumps({**CHAT, "tools": marker}))) + output = tmp_path / "reading" + + with caplog.at_level("WARNING", logger=reading_result.__name__): + assert _run(tmp_path) == 78 + + assert _failure_in(output)["details"] == {"class": "ValidationError", "code": ""} + [logged] = [record for record in caplog.records if record.name == reading_result.__name__] + assert logged.exc_text is None + assert "hunter2" not in caplog.text + + +def test_the_contract_checked_is_the_contract_used(tmp_path, model, monkeypatch): + source = _repo(tmp_path) + (source / "agents/graph").mkdir(parents=True) + (source / "agents/graph/graph.py").write_text( + "from langgraph.graph import StateGraph\nStateGraph(int)\n", encoding="utf-8" + ) + checked = {**CHAT, "runtime": {"language": "python", "workdir": SUPPORT}} + model(_writes(checked)) + output = tmp_path / "reading" + real = reading_result.named_framework_folder + + def a_late_save_lands(*args, **kwargs): + # The session saves again before the folder is chosen: another folder, over the cap. + (output / "contract.json").write_text( + json.dumps( + { + **CHAT, + "runtime": {"language": "python", "workdir": "agents/graph"}, + "real_use_cases": ["x" * (6 * MIB)], + } + ), + encoding="utf-8", + ) + return real(*args, **kwargs) + + monkeypatch.setattr(reading_result, "named_framework_folder", a_late_save_lands) + + assert _run(tmp_path) == 0 + + agent = json.loads((output / "agent.json").read_text(encoding="utf-8"))["agent"] + # The checked contract's folder and framework, not the late one's. + assert agent == build_authored_agent( + AgentContract.model_validate(checked).model_dump(mode="json"), + framework=detect_framework(source / SUPPORT, repo_root=source, package_dir=None), + ) + assert detect_framework(source / "agents/graph", repo_root=source, package_dir=None) != ( + detect_framework(source / SUPPORT, repo_root=source, package_dir=None) + ) + + +def test_an_environment_backed_job_is_read(tmp_path): + job = HarnessJob.model_validate( + {**_job(), "agent": {"connector": "vapi", "mode": "environment_backed", "config": {}}} + ) + + assert agent_reading._check_job(job) is None + + +_CLOSE_CHILD = r""" +import asyncio, dataclasses, json, os, signal, sys, time +config = json.loads(sys.argv[1]) +sys.path[:0] = config["sys_path"] +from pathlib import Path +from fi.alk.harness import agent_reading, reading_entry, session +from fi.alk.harness.backends import StageDone +from fi.alk.harness.reading_entry import run_reading_mode + +reading_entry._SECRET_FILES = () +root = Path(config["root"]) +output = root / "reading" + + +class Live: + def __init__(self, spec): + self.spec = spec + + async def start(self): + pass + + async def send(self, message): + pass + + async def interrupt(self): + if config["interrupt"] == "swallows": + # Swallows the first cancel, then hangs on; only the loop's shutdown ends it. + try: + await asyncio.sleep(3600) + except asyncio.CancelledError: + pass + await asyncio.sleep(3600) + if config["interrupt"] == "sigint": + # A Ctrl-C that lands while the stop is being delivered. + os.kill(os.getpid(), signal.SIGINT) + await asyncio.sleep(3600) + return True + + async def stop(self): + if config["close"] != "hangs": + return + # A close that swallows every cancel, the loop's own shutdown's included. + while True: + try: + await asyncio.sleep(3600) + except asyncio.CancelledError: + pass + + async def replies(self): + if config["contract"] is not None: + (output / "contract.json").write_text(json.dumps(config["contract"])) + if config.get("after_save") == "sigint": + # A Ctrl-C just after the save, before the reading has decided anything. + os.kill(os.getpid(), signal.SIGINT) + await asyncio.Event().wait() + yield StageDone(outcome="success", turns=1, cost_usd=0.0) + + +class Backend: + name = "vertex-gemini" + default_model = "gemini-2.5-flash" + + def create(self, spec): + return Live(spec) + + def can_drive(self, model): + return True + + +session.resolve = lambda *a, **k: Backend() +mode = dataclasses.replace( + agent_reading._read_agent(interrupt_wait=0.2), limit_margin_seconds=config["margin"] +) +(root / "started").write_text(repr(time.time())) +sys.exit(run_reading_mode(mode, job_path=root / "job.json", source=root / "source", output=output)) +""" + + +def _read_in_a_child(root: Path, *, soft: float, margin: float, **behaviour): + _repo(root) + (root / "job.json").write_text(json.dumps(_job()), encoding="utf-8") + config = { + "sys_path": [entry for entry in sys.path if entry], + "root": str(root), + "contract": CHAT, + "margin": margin, + "interrupt": "honours", + "close": "ends", + **behaviour, + } + env = { + **os.environ, + _LIMIT_ENV: str(soft), + "ALK_HARNESS_MODEL": "gemini-2.5-flash", + "OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:9", + "OTEL_TRACES_EXPORTER": "none", + } + env.pop(spend.JOURNAL_ALIAS, None) + finished = subprocess.run( + [sys.executable, "-c", _CLOSE_CHILD, json.dumps(config)], + env=env, + capture_output=True, + text=True, + timeout=60, + ) + return finished, float((root / "started").read_text(encoding="utf-8")) + + +def test_a_close_that_never_ends_still_leaves_the_document_well_inside_the_hard_limit(tmp_path): + soft, margin = 0.5, 6.0 + + finished, started = _read_in_a_child(tmp_path, soft=soft, margin=margin, close="hangs") + + output = tmp_path / "reading" + assert finished.returncode == 0, finished.stderr + assert not (output / "failure.json").exists() + # Decided at soft + interrupt + wait + close (about 0.9 s), long before the hard limit + # (soft + margin); only the shutdown the close blocks is ended by the limit, with the code. + documented = (output / "agent.json").stat().st_mtime - started + assert documented < soft + margin - 3.0 + assert _cost_in(output)["complete"] is False + + +def test_an_interrupt_that_swallows_its_cancel_does_not_hold_the_reading(tmp_path): + soft, margin = 0.5, 6.0 + + finished, started = _read_in_a_child(tmp_path, soft=soft, margin=margin, interrupt="swallows") + + output = tmp_path / "reading" + assert finished.returncode == 0, finished.stderr + assert not (output / "failure.json").exists() + assert (output / "agent.json").stat().st_mtime - started < soft + margin - 3.0 + + +def test_a_ctrl_c_during_the_stop_passes_through(tmp_path): + finished, _ = _read_in_a_child( + tmp_path, soft=0.5, margin=6.0, interrupt="sigint", contract=None + ) + + output = tmp_path / "reading" + assert finished.returncode in (-signal.SIGINT, 130), finished.stderr + assert "KeyboardInterrupt" in finished.stderr + # Not a timeout: an interrupt ends the reading as an interrupt, before any save. + assert not (output / "failure.json").exists() + assert _cost_in(output)["complete"] is False + + +def test_a_ctrl_c_after_the_save_passes_through_and_leaves_the_files(tmp_path): + finished, _ = _read_in_a_child(tmp_path, soft=5.0, margin=6.0, after_save="sigint") + + output = tmp_path / "reading" + assert finished.returncode in (-signal.SIGINT, 130), finished.stderr + assert "KeyboardInterrupt" in finished.stderr + # The saved contract is left as it is; nothing is written over it or after it. + assert (output / "contract.json").exists() + assert not (output / "failure.json").exists() + assert not (output / "agent.json").exists() + + +@pytest.mark.skipif(sys.version_info < (3, 11), reason="exception groups arrived in 3.11") +def test_a_group_of_cancels_before_the_session_opens_is_a_crash_with_its_file(tmp_path, model): + _repo(tmp_path) + fake = model() + group = getattr(builtins, "BaseExceptionGroup") + # The shape an anyio task group gives when a child task is cancelled. + fake.start_error = group("the session's tasks failed", [asyncio.CancelledError()]) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_crashed", True) + assert failure["message"].startswith("BaseExceptionGroup") + _assert_cost(output, 0.0, False) + + +def test_the_longest_stop_leaves_room_for_the_document_inside_the_margin(): + # The stop's own bounds, exactly: the interrupt, the wait after it, and the close. + bounds = ( + agent_reading._INTERRUPT_SECONDS, + agent_reading._INTERRUPT_WAIT_SECONDS, + agent_reading._CLOSE_SECONDS, + ) + assert bounds == (10.0, 60.0, 20.0) + assert sum(bounds) == 90.0 + # Then the saved contract's document, whose framework scan can take over a minute on a + # crafted repository: at least 150 s of the margin stays for it. + assert sum(bounds) + 150.0 < agent_reading._HARD_LIMIT_MARGIN_SECONDS + assert agent_reading.READ_AGENT.limit_margin_seconds == agent_reading._HARD_LIMIT_MARGIN_SECONDS + + +def test_a_url_the_redaction_cannot_parse_leaves_no_password_anywhere( + tmp_path, model, capfd, caplog +): + _repo(tmp_path) + leaky = "https://admin:hunter2\uff03x@db.example/" + model(_writes({**CHAT, "real_use_cases": [leaky]})) + output = tmp_path / "reading" + + with caplog.at_level("DEBUG"): + assert _run(tmp_path) == 0 + + kept = (output / "agent.json").read_text(encoding="utf-8") + captured = capfd.readouterr() + assert not (output / "failure.json").exists() + assert "hunter2" not in kept + captured.out + captured.err + caplog.text + + +def test_a_failed_document_is_logged_with_its_masked_trace(tmp_path, model, monkeypatch, caplog): + secret = "sk-document-0123456789" + secrets = tmp_path / "secrets.json" + secrets.write_text(json.dumps({"KEY": secret}), encoding="utf-8") + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (secrets,)) + + def breaks(*_args, **_kwargs): + raise RuntimeError(f"the document builder broke on {secret}") + + monkeypatch.setattr(scenario_source, "build_authored_agent", breaks) + _repo(tmp_path) + model(_writes(CHAT)) + + with caplog.at_level("WARNING", logger=reading_result.__name__): + assert _run(tmp_path) == 78 + + [logged] = [record for record in caplog.records if record.name == reading_result.__name__] + assert "RuntimeError: the document builder broke on ***" in logged.exc_text + assert secret not in caplog.text + + +def test_a_folder_mismatch_is_an_invalid_document(tmp_path, model): + _repo(tmp_path) + # Saved with no folder recorded, for a job that names one. + model(_writes(CHAT, cost_usd=0.11)) + output = tmp_path / "reading" + + assert _run(tmp_path, _job(subfolder=SUPPORT)) == 78 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_document_invalid", False) + # The build's own code for it, from the build's own document code. + assert failure["details"] == { + "class": "AgentBuildError", + "code": "agent_invalid", + "detail": "subfolder_mismatch", + } + assert (output / "contract.json").exists() + assert not (output / "agent.json").exists() + _assert_cost(output, 0.11, True) + + +def _stored_size(inner: dict) -> int: + return len(json.dumps(inner, ensure_ascii=False).encode("utf-8")) + + +def _body_size(inner: dict) -> int: + return len( + json.dumps( + {"sdk_version": fi.alk.__version__, "agent": inner}, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + ) + + +def _sized(target: int, measure, items: int) -> dict: + """An inner document that ``measure`` puts at exactly ``target`` bytes.""" + + def inner(pad: int) -> dict: + return {"items": [0] * items, "pad": "a" * pad} + + return inner(target - measure(inner(0))) + + +@pytest.mark.parametrize( + ("inner", "passes"), + [ + # Many separators: over by the platform's stored measure, under by the file's. + (lambda: _sized(MIB + 1, _stored_size, 300_000), False), + (lambda: _sized(MIB, _stored_size, 300_000), True), + (lambda: _sized(MIB + 1, _body_size, 0), False), + ], + ids=["stored_over", "stored_exactly", "file_over"], +) +def test_an_oversized_document_is_refused(tmp_path, model, monkeypatch, inner, passes): + _repo(tmp_path) + document = inner() + if not passes and document["items"]: + assert _body_size(document) <= MIB + monkeypatch.setattr(scenario_source, "build_authored_agent", lambda *a, **k: document) + model(_writes(CHAT, cost_usd=0.05)) + output = tmp_path / "reading" + + code = _run(tmp_path) + + if passes: + assert code == 0 + assert (output / "agent.json").stat().st_size == _body_size(document) + else: + assert code == 78 + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_too_large", False) + assert not (output / "agent.json").exists() + assert (output / "contract.json").exists() + _assert_cost(output, 0.05, True) + + +@pytest.mark.parametrize(("size", "passes"), [(5 * MIB + 1, False), (5 * MIB, True)]) +def test_an_oversized_contract_is_refused(tmp_path, model, size, passes): + _repo(tmp_path) + text = json.dumps(CHAT) + # Trailing spaces still parse, so only the file's size changes. + model(_writes(text=text + " " * (size - len(text.encode("utf-8"))), cost_usd=0.05)) + output = tmp_path / "reading" + + code = _run(tmp_path) + + assert (output / "contract.json").stat().st_size == size + if passes: + assert code == 0 + assert (output / "agent.json").exists() + else: + assert code == 78 + assert _failure_in(output)["code"] == "agent_reading_too_large" + assert not (output / "agent.json").exists() + _assert_cost(output, 0.05, True) + + +# -- R10 to R19: how a turn's ending is read ---------------------------------------------------- + + +def test_no_contract_after_the_reminder_is_incomplete(tmp_path, model): + _repo(tmp_path) + # Claude's results carry the session's running total. + fake = model(_ends(cost_usd=0.07875), _ends(cost_usd=0.1575)) + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + assert len(fake.sent) == 2 + assert fake.sent[1] == SUBMIT_REMINDER + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_incomplete", False) + assert failure["details"] == {"outcome": "success", "turns": 2} + _assert_cost(output, 0.1575, True) + + +# Gemini's call limit arrives as a raise, so its own test below drives the real backend; a +# stream that ends with no final answer is the outcome here. One turn, well under the budget, +# so only the outcome can mark it. +_TURN_LIMIT_SHAPES = [ + ("claude", {"outcome": "error_max_turns", "is_error": True}), + ("vertex-gemini", {"outcome": "max_turns", "is_error": True, "turns": 1}), +] + + +@pytest.mark.parametrize(("name", "ending"), _TURN_LIMIT_SHAPES, ids=["claude", "gemini"]) +def test_the_turn_limit_is_incomplete_not_a_model_error(tmp_path, model, name, ending): + _repo(tmp_path) + fake = model(_ends(cost_usd=0.2, **ending), name=name) + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + assert len(fake.sent) == 1 + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_incomplete", False) + assert failure["details"]["outcome"] == ending["outcome"] + _assert_cost(output, 0.2, True) + + +@pytest.mark.parametrize("status", [408, 409, 429, 500, 529]) +def test_a_provider_error_is_retryable(tmp_path, model, status): + _repo(tmp_path) + model(_ends(cost_usd=0.3, is_error=True, api_error_status=status, errors=["overloaded"])) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_model_error", True) + assert failure["details"] == {"status": status} + assert "overloaded" in failure["message"] + _assert_cost(output, 0.3, True) + + +@pytest.mark.parametrize( + ("name", "ending"), + [ + ("claude", {"outcome": "success", "is_error": True}), + # A provider exception, well inside the call budget. + ("vertex-gemini", {"outcome": "failed", "is_error": True, "turns": 1}), + ], + ids=["claude", "gemini"], +) +def test_a_failure_with_no_status_is_a_retryable_model_error(tmp_path, model, name, ending): + _repo(tmp_path) + fake = model(_ends(**ending), _ends(**ending), name=name) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + assert len(fake.sent) == 2 + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_model_error", True) + assert failure["details"] == {"status": None} + # A request cut with no status may be billed without a total reaching the stage. + assert _cost_in(output)["complete"] is False + + +@pytest.mark.parametrize("status", [400, 401, 404]) +def test_a_refusing_4xx_is_not_retried(tmp_path, model, status): + _repo(tmp_path) + model(_ends(cost_usd=0.01, is_error=True, api_error_status=status)) + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_model_error", False) + assert failure["details"] == {"status": status} + + +def test_a_gemini_grpc_code_is_retryable(tmp_path, model): + _repo(tmp_path) + model( + _ends(outcome="failed", is_error=True, api_error_status=14), name="vertex-gemini" + ) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_model_error", True) + assert failure["details"] == {"status": 14} + + +def test_a_failed_opening_gets_no_reminder(tmp_path, model): + _repo(tmp_path) + fake = model(_ends(is_error=True, api_error_status=529)) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + assert len(fake.sent) == 1 + assert _failure_in(output)["code"] == "agent_reading_model_error" + + +@pytest.mark.parametrize(("name", "ending"), _TURN_LIMIT_SHAPES, ids=["claude", "gemini"]) +def test_a_turn_limit_opening_gets_no_reminder(tmp_path, model, name, ending): + _repo(tmp_path) + fake = model(_ends(**ending), name=name) + + _run(tmp_path) + + assert len(fake.sent) == 1 + + +def test_an_error_flag_without_a_status_still_gets_the_reminder(tmp_path, model): + _repo(tmp_path) + fake = model( + _ends(outcome="success", is_error=True, cost_usd=0.1), + _writes(CHAT, cost_usd=0.2), + ) + output = tmp_path / "reading" + + assert _run(tmp_path) == 0 + + assert fake.sent[1:] == [SUBMIT_REMINDER] + assert (output / "agent.json").exists() + # The first turn's error had no status, so its spend may not all be in the total. + _assert_cost(output, 0.2, False) + + +# Priced, so the stop's cost is checked against the table and not left at zero. +_PRICED_GEMINI = "gemini-3.7-flash" + + +def _real_gemini(monkeypatch, reply) -> tuple[list[str], list[str]]: + """The real Gemini backend and ADK runner; only the model is an in-memory one. + + ``reply(who, calls)`` gives the reply to the agent named ``who``; ``calls`` names every + agent that has called so far. Returns the messages sent and the calls made. + """ + from google.adk.models.base_llm import BaseLlm + + from fi.alk.harness.backends import vertex_gemini + + calls: list[str] = [] + + class Scripted(BaseLlm): + async def generate_content_async(self, llm_request, stream=False): + labels = (llm_request.config.labels or {}) if llm_request.config else {} + calls.append(labels.get("adk_agent_name", "")) + yield reply(calls[-1], calls) + + # The session writes these into the environment; set here first, they are put back after. + monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "test-project") + monkeypatch.setenv("GOOGLE_GENAI_USE_VERTEXAI", "TRUE") + monkeypatch.setenv("GOOGLE_CLOUD_LOCATION", "global") + monkeypatch.setenv("ALK_HARNESS_MODEL", _PRICED_GEMINI) + monkeypatch.setattr(vertex_gemini, "_llm", lambda name: Scripted(model=name)) + backend = vertex_gemini.VertexGeminiBackend() + monkeypatch.setattr(session, "resolve", lambda *a, **k: backend) + sent: list[str] = [] + real_send = vertex_gemini.VertexGeminiSession.send + + async def send(self, message): + sent.append(message) + return await real_send(self, message) + + monkeypatch.setattr(vertex_gemini.VertexGeminiSession, "send", send) + return sent, calls + + +def _gemini_usage(): + from google.genai import types + + return types.GenerateContentResponseUsageMetadata( + prompt_token_count=1000, candidates_token_count=10 + ) + + +def _gemini_calls(name: str, args: dict, n: int): + from google.adk.models.llm_response import LlmResponse + from google.genai import types + + call = types.FunctionCall(name=name, args=args, id=f"call-{n}") + return LlmResponse( + content=types.Content(role="model", parts=[types.Part(function_call=call)]), + usage_metadata=_gemini_usage(), + ) + + +def _gemini_priced(calls: int) -> float: + # The fake's 1,000 tokens in and 10 out per call, at the table's own rates, kept to the + # six places the spend record keeps. + from fi.alk.harness.backends.vertex_gemini import PRICES_PER_MILLION + + rate_in, rate_out = PRICES_PER_MILLION[_PRICED_GEMINI][:2] + return round(calls * (1000 * rate_in + 10 * rate_out) / 1_000_000, 6) + + +def test_a_gemini_call_limit_is_incomplete_with_no_reminder(tmp_path, model, monkeypatch): + sent, calls = _real_gemini( + monkeypatch, lambda who, calls: _gemini_calls("Glob", {"pattern": "*.py"}, len(calls)) + ) + _few_turns(monkeypatch, 3) + _repo(tmp_path) + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + assert len(sent) == 1 + assert len(calls) == 3 + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_incomplete", False) + # The backend's own count reaches the budget when ADK stops the calls. + assert failure["details"] == {"outcome": "failed", "turns": 3} + # ADK stops before the call it refuses, so every call made is in the total. + _assert_cost(output, _gemini_priced(3), True) + + +def test_a_gemini_call_limit_after_a_malformed_worker_reply_gets_no_reminder( + tmp_path, model, monkeypatch +): + from google.adk.models.llm_response import LlmResponse + + def reply(who, calls): + if who != "agent_reader": + return _gemini_calls("agent_reader", {"request": "Trace the add tool."}, len(calls)) + if calls.count("agent_reader") == 1: + # Gemini's malformed call: no content, so the backend doesn't count it as a turn, + # but ADK still spends a call on it. + return LlmResponse( + error_code="MALFORMED_FUNCTION_CALL", + error_message="Malformed function call", + usage_metadata=_gemini_usage(), + ) + return _gemini_calls("Glob", {"pattern": "*.py"}, len(calls)) + + sent, calls = _real_gemini(monkeypatch, reply) + _few_turns(monkeypatch, 6) + _repo(tmp_path) + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + # One budget, not two: no reminder opened a second one. + assert len(sent) == 1 + assert len(calls) == 6 + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_incomplete", False) + # The backend's count falls short of the budget; ADK's own words mark the stop. + assert failure["details"]["turns"] < 6 + _assert_cost(output, _gemini_priced(6), True) + + +def test_an_earlier_failed_turn_is_still_a_model_error(tmp_path, model): + _repo(tmp_path) + fake = model(_ends(outcome="success", is_error=True), _ends()) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + assert len(fake.sent) == 2 + assert _failure_in(output)["code"] == "agent_reading_model_error" + + +def test_a_cut_on_the_last_allowed_call_is_not_a_turn_limit(tmp_path, model): + _repo(tmp_path) + fake = model( + _ends(outcome="failed", is_error=True, turns=70, errors=["connection reset"]), + _ends(), + name="vertex-gemini", + ) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + assert fake.sent[1:] == [SUBMIT_REMINDER] + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_model_error", True) + # The cut request may be billed without its total reaching the stage. + assert _cost_in(output)["complete"] is False + + +@pytest.mark.parametrize( + "said", + [ + "Tool 'Max number of llm calls limit of `6` exceeded' not found", + "invalid literal for int() with base 10: 'Max number of llm calls limit of `6` exceeded'", + "Max number of llm calls limit of `6` exceeded, said the model", + ], + ids=["tool-name", "offset", "trailing"], +) +def test_the_call_limit_phrase_inside_other_text_is_a_model_error(tmp_path, model, said): + _repo(tmp_path) + fake = model( + _ends(outcome="failed", is_error=True, errors=[said]), _ends(), name="vertex-gemini" + ) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + assert len(fake.sent) == 2 + assert _failure_in(output)["code"] == "agent_reading_model_error" + + +@pytest.mark.parametrize( + ("name", "cut", "limit"), + [ + ("claude", {"outcome": "success", "is_error": True}, _TURN_LIMIT_SHAPES[0][1]), + ( + "vertex-gemini", + {"outcome": "failed", "is_error": True}, + _TURN_LIMIT_SHAPES[1][1], + ), + ( + "vertex-gemini", + {"outcome": "failed", "is_error": True}, + { + "outcome": "failed", + "is_error": True, + "errors": ["Max number of llm calls limit of `200` exceeded"], + }, + ), + ], + ids=["claude", "gemini", "gemini-call-limit"], +) +def test_a_turn_limit_on_the_reminder_wins_over_an_earlier_cut( + tmp_path, model, name, cut, limit +): + _repo(tmp_path) + fake = model(_ends(**cut), _ends(**limit), name=name) + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + assert len(fake.sent) == 2 + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_incomplete", False) + # The opening's cut may still be billed without its total reaching the stage. + assert _cost_in(output)["complete"] is False + + +def test_a_cut_on_the_reminder_leaves_the_cost_incomplete(tmp_path, model): + _repo(tmp_path) + fake = model(_ends(cost_usd=0.07875), _ends(outcome="success", is_error=True)) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + assert len(fake.sent) == 2 + assert _failure_in(output)["code"] == "agent_reading_model_error" + assert _cost_in(output)["complete"] is False + + +# -- R20 to R27: a quiet stream, the soft limit, and what was saved ------------------------------ + + +def test_an_idle_stream_is_not_replayed(tmp_path, model, monkeypatch): + _repo(tmp_path) + monkeypatch.setattr(session, "STAGE_IDLE_RETRIES", 1) + _short_idle(monkeypatch) + fake = model(_quiet) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + assert len(fake.created) == 1 + assert fake.interrupts == [] + # Only this stage skips the replay; the process's own setting is left as it was. + assert session.STAGE_IDLE_RETRIES == 1 + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_model_error", True) + _assert_cost(output, 0.0, False) + + +def test_a_contract_saved_before_an_idle_stream_is_kept(tmp_path, model, monkeypatch): + _repo(tmp_path) + _short_idle(monkeypatch) + + async def saves_then_goes_quiet(live): + _save(live.backend.output, CHAT) + await asyncio.Event().wait() + yield _done() + + model(saves_then_goes_quiet) + output = tmp_path / "reading" + + assert _run(tmp_path) == 0 + + assert (output / "agent.json").exists() + assert not (output / "failure.json").exists() + _assert_cost(output, 0.0, False) + + +def test_the_soft_limit_interrupts_once_and_charges_what_arrives(tmp_path, model, monkeypatch): + _repo(tmp_path) + _short_limit(monkeypatch) + + async def answers_the_interrupt(live): + await live.interrupted.wait() + yield _done(outcome="error_during_execution", is_error=True, cost_usd=0.42) + + fake = model(answers_the_interrupt) + output = tmp_path / "reading" + + assert _run(tmp_path, wait=2.0) == 78 + + assert len(fake.sent) == 1 + assert len(fake.interrupts) == 1 + assert _failure_in(output)["code"] == "agent_reading_timed_out" + _assert_cost(output, 0.42, False) + + +def test_the_soft_limit_marks_cost_incomplete_when_nothing_arrives(tmp_path, model, monkeypatch): + _repo(tmp_path) + _short_limit(monkeypatch) + fake = model(_quiet) + output = tmp_path / "reading" + + assert _run(tmp_path, wait=0.3) == 78 + + assert len(fake.interrupts) == 1 + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_timed_out", False) + _assert_cost(output, 0.0, False) + + +def test_an_interrupt_that_raises_is_still_a_timeout(tmp_path, model, monkeypatch, caplog): + _repo(tmp_path) + _short_limit(monkeypatch) + fake = model(_quiet) + fake.interrupt_raises = True + output = tmp_path / "reading" + + with caplog.at_level("WARNING", logger=agent_reading.__name__): + assert _run(tmp_path, wait=0.3) == 78 + + assert _failure_in(output)["code"] == "agent_reading_timed_out" + [logged] = [ + record + for record in caplog.records + if record.name == agent_reading.__name__ and "not delivered" in record.getMessage() + ] + assert "RuntimeError: the interrupt control request timed out" in logged.exc_text + + +def test_the_readings_own_log_is_masked(tmp_path, model, monkeypatch, caplog): + secret = "sk-provider-0123456789" + secrets = tmp_path / "secrets.json" + secrets.write_text(json.dumps({"TARGET_KEY": secret}), encoding="utf-8") + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (secrets,)) + _repo(tmp_path) + _short_limit(monkeypatch) + fake = model(_quiet) + fake.interrupt_raises = True + fake.interrupt_says = f"the provider refused key {secret}" + + with caplog.at_level("WARNING", logger=agent_reading.__name__): + assert _run(tmp_path, wait=0.3) == 78 + + [logged] = [ + record + for record in caplog.records + if record.name == agent_reading.__name__ and "not delivered" in record.getMessage() + ] + assert "RuntimeError: the provider refused key ***" in logged.exc_text + assert secret not in caplog.text + + +def test_a_session_that_breaks_after_the_stop_is_logged_and_still_a_timeout( + tmp_path, model, monkeypatch, caplog +): + _repo(tmp_path) + _short_limit(monkeypatch) + + async def breaks_when_interrupted(live): + await live.interrupted.wait() + raise ConnectionResetError("the stream broke after the stop") + yield _done() + + model(breaks_when_interrupted) + output = tmp_path / "reading" + + with caplog.at_level("WARNING", logger=agent_reading.__name__): + assert _run(tmp_path, wait=1.0) == 78 + + assert _failure_in(output)["code"] == "agent_reading_timed_out" + [logged] = [ + record + for record in caplog.records + if record.name == agent_reading.__name__ and "ended with an error" in record.getMessage() + ] + assert "ConnectionResetError: the stream broke after the stop" in logged.exc_text + + +def test_a_contract_saved_at_the_soft_limit_is_kept(tmp_path, model, monkeypatch): + _repo(tmp_path) + _short_limit(monkeypatch) + + async def saves_when_interrupted(live): + await live.interrupted.wait() + _save(live.backend.output, CHAT) + yield _done(outcome="error_during_execution", is_error=True, cost_usd=0.3) + + model(saves_when_interrupted) + output = tmp_path / "reading" + + assert _run(tmp_path, wait=2.0) == 0 + + assert (output / "agent.json").exists() + assert not (output / "failure.json").exists() + _assert_cost(output, 0.3, False) + + +def test_nothing_is_saved_after_the_soft_limit_decides(tmp_path, model, monkeypatch): + _repo(tmp_path) + _short_limit(monkeypatch, soft=0.3) + + async def saves_too_late(live): + # Ignores the interrupt, and would save only after the wait has run out. + await asyncio.sleep(1.5) + await _gate(live, CHAT) + yield _done() + + fake = model(saves_too_late) + output = tmp_path / "reading" + + assert _run(tmp_path, wait=0.3) == 78 + + # The session was closed before either result file could be written. + assert fake.stopped == [{"failure": False, "agent": False}] + assert not (output / "contract.json").exists() + assert _failure_in(output)["code"] == "agent_reading_timed_out" + + +def test_a_saved_contract_wins_over_an_error_flag(tmp_path, model): + _repo(tmp_path) + model(_writes(CHAT, is_error=True, cost_usd=0.1)) + output = tmp_path / "reading" + + assert _run(tmp_path) == 0 + + assert (output / "agent.json").exists() + assert not (output / "failure.json").exists() + + +def test_a_turn_with_no_ending_leaves_the_cost_incomplete(tmp_path, model): + _repo(tmp_path) + + async def saves_and_never_reports(live): + _save(live.backend.output, CHAT) + for nothing in (): + yield nothing + + fake = model(saves_and_never_reports) + output = tmp_path / "reading" + + assert _run(tmp_path) == 0 + + assert len(fake.sent) == 1 + assert (output / "agent.json").exists() + _assert_cost(output, 0.0, False) + + +# -- R28 to R32: what is charged ---------------------------------------------------------------- + + +def test_the_cost_follows_each_turn(tmp_path, model): + _repo(tmp_path) + + async def reads_the_cost(live): + cost = _cost_in(live.backend.output) + live.backend.seen.append((cost["total_usd"], cost["complete"])) + yield _done(cost_usd=0.1575) + + fake = model(_ends(cost_usd=0.07875), reads_the_cost) + + _run(tmp_path) + + assert fake.seen == [(pytest.approx(0.07875), False)] + + +def _stage_entry() -> dict: + [entry] = spend.snapshot()["stages"] + return entry + + +def test_claude_running_totals_are_counted_once(tmp_path, model): + _repo(tmp_path) + model( + _ends(cost_usd=0.07875, tokens_in=100_000), + _ends(cost_usd=0.1575, tokens_in=200_000), + ) + output = tmp_path / "reading" + + _run(tmp_path) + + _assert_cost(output, 0.1575, True) + assert _stage_entry()["tokens_in"] == 200_000 + + +def test_gemini_counts_are_added_as_reported(tmp_path, model): + _repo(tmp_path) + model( + _ends(cost_usd=0.1, tokens_in=1_000), + _ends(cost_usd=0.1, tokens_in=1_000), + name="vertex-gemini", + ) + output = tmp_path / "reading" + + _run(tmp_path) + + _assert_cost(output, 0.2, True) + assert _stage_entry()["tokens_in"] == 2_000 + + +class _MessagesStub(BaseHTTPRequestHandler): + """The Messages API on 127.0.0.1, answering every request with fixed usage.""" + + seen: list[str] = [] + + def log_message(self, *args) -> None: + pass + + def _json(self, body: dict) -> None: + raw = json.dumps(body).encode() + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + + def do_HEAD(self) -> None: + self.send_response(200) + self.end_headers() + + def do_GET(self) -> None: + self._json({"data": [], "has_more": False}) + + def do_POST(self) -> None: + raw = self.rfile.read(int(self.headers.get("content-length") or 0)) + try: + request = json.loads(raw) + except ValueError: + request = {} + self.seen.append(self.path) + if "count_tokens" in self.path: + self._json({"input_tokens": 10}) + return + model_name = request.get("model", "claude-sonnet-4-5") + if not request.get("stream"): + self._json( + { + "id": "msg_x", + "type": "message", + "role": "assistant", + "model": model_name, + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 1}, + } + ) + return + self.send_response(200) + self.send_header("content-type", "text/event-stream") + self.send_header("cache-control", "no-cache") + self.end_headers() + + def event(name: str, data: dict) -> None: + self.wfile.write(f"event: {name}\ndata: {json.dumps(data)}\n\n".encode()) + self.wfile.flush() + + event( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": model_name, + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": { + "input_tokens": 100_000, + "output_tokens": 1, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + }, + }, + ) + event( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ) + event( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "done"}}, + ) + event("content_block_stop", {"type": "content_block_stop", "index": 0}) + event( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 1000}, + }, + ) + event("message_stop", {"type": "message_stop"}) + + +def test_the_claude_cli_reports_running_totals(tmp_path): + import claude_agent_sdk + from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient, ResultMessage + + cli = Path(claude_agent_sdk.__file__).parent / "_bundled" / "claude" + if not cli.is_file(): + pytest.fail(f"the Claude CLI bundled with claude-agent-sdk was not found at {cli}") + _MessagesStub.seen = [] + server = ThreadingHTTPServer(("127.0.0.1", 0), _MessagesStub) + threading.Thread(target=server.serve_forever, daemon=True).start() + nowhere = "http://127.0.0.1:9" + env = { + "ANTHROPIC_BASE_URL": f"http://127.0.0.1:{server.server_address[1]}", + "ANTHROPIC_API_KEY": "placeholder-not-a-key", + "ANTHROPIC_AUTH_TOKEN": "", + # Another test in the same process may leave these set, and the CLI would go there. + "CLAUDE_CODE_USE_VERTEX": "", + "CLAUDE_CODE_USE_BEDROCK": "", + "CLAUDE_CODE_USE_FOUNDRY": "", + "ANTHROPIC_VERTEX_PROJECT_ID": "", + "ANTHROPIC_VERTEX_BASE_URL": nowhere, + "ANTHROPIC_BEDROCK_BASE_URL": nowhere, + "ANTHROPIC_FOUNDRY_BASE_URL": nowhere, + "OTEL_EXPORTER_OTLP_ENDPOINT": nowhere, + "CLAUDE_CODE_ENABLE_TELEMETRY": "0", + "DISABLE_TELEMETRY": "1", + "DISABLE_ERROR_REPORTING": "1", + "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1", + "DISABLE_AUTOUPDATER": "1", + "CLAUDE_CONFIG_DIR": str(tmp_path / "config"), + } + work = tmp_path / "work" + work.mkdir() + options = ClaudeAgentOptions( + model="claude-sonnet-4-5", + max_turns=5, + env=env, + cwd=str(work), + allowed_tools=[], + setting_sources=[], + cli_path=str(cli), + ) + results: list = [] + + async def two_queries() -> None: + async with ClaudeSDKClient(options=options) as client: + for prompt in ("first", "second, as the reminder would be"): + await client.query(prompt) + async for message in client.receive_response(): + if isinstance(message, ResultMessage): + results.append(message) + + try: + asyncio.run(asyncio.wait_for(two_queries(), 120)) + finally: + server.shutdown() + server.server_close() + + asked = [path for path in _MessagesStub.seen if "count_tokens" not in path] + assert len(asked) == 2, asked + first, second = results + assert (first.is_error, second.is_error) == (False, False) + assert first.total_cost_usd > 0 + assert second.total_cost_usd == pytest.approx(2 * first.total_cost_usd) + + def input_tokens(result) -> int: + return sum( + int((usage or {}).get("inputTokens", 0)) + for usage in (result.model_usage or {}).values() + ) + + assert [input_tokens(first), input_tokens(second)] == [100_000, 200_000] + + +def test_a_forbidden_model_is_not_retried(tmp_path, model): + _repo(tmp_path) + model(_ends(cost_usd=0.2, models={"claude-sonnet-4-5"})) + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_model_error", False) + # Spent before the stop, and that turn never finished being kept. + _assert_cost(output, 0.2, False) + + +# -- R33 to R36: the stage the reading runs ----------------------------------------------------- + + +_EVALS = [{"name": "tone_check", "description": "Checks the tone.", "modality": "chat"}] + + +class _Opened(Exception): + pass + + +@pytest.mark.parametrize( + ("metadata", "interactive", "expected"), + [ + ( + {"generic_harness_v1": True, "available_evals": _EVALS}, + False, + {"available_evals": _EVALS, "require_setup_contract": True, "max_turns": 140}, + ), + ( + {"available_evals": _EVALS}, + False, + {"available_evals": _EVALS, "require_setup_contract": False, "max_turns": 70}, + ), + ( + {"generic_harness_v1": True}, + True, + {"available_evals": None, "require_setup_contract": False, "max_turns": 70}, + ), + ( + # Only the flag itself, not a value that merely reads as true. + {"generic_harness_v1": "true"}, + False, + {"available_evals": None, "require_setup_contract": False, "max_turns": 70}, + ), + ], + ids=["generic", "not_generic", "generic_with_an_operator", "flag_as_text"], +) +def test_the_build_opens_its_reading_as_it_always_has( + tmp_path, monkeypatch, metadata, interactive, expected +): + from fi.alk.harness import cli + + job = HarnessJob.model_validate(_job(metadata=metadata)) + assert reading_settings(job, interactive=interactive) == expected + seen: dict = {} + + def opened(_source, **kwargs): + seen.update(kwargs) + raise _Opened + + monkeypatch.setattr(cli, "open_stage", opened) + args = argparse.Namespace( + job=job, + kind="repo", + name="agent", + path=str(_repo(tmp_path)), + out=str(tmp_path / "out"), + interactive=interactive, + ) + + with pytest.raises(_Opened): + asyncio.run(cli._understand(args)) + + assert {key: seen[key] for key in expected} == expected + + +@pytest.mark.parametrize(("generic", "turns"), [(False, 70), (True, 140)]) +def test_the_reading_opens_as_the_build_does(tmp_path, model, generic, turns): + _repo(tmp_path) + fake = model() + metadata = {"available_evals": _EVALS, **({"generic_harness_v1": True} if generic else {})} + + _run(tmp_path, _job(metadata=metadata)) + + [spec] = fake.created + assert spec.max_turns == turns + assert spec.model == chosen_model() + assert spec.thinking is True + assert "tone_check" in spec.system_prompt + assert CONTRACT_SERVER in spec.servers + + +def test_the_reading_opens_with_the_builds_words_and_the_jobs_guidance(tmp_path, model): + from fi.alk.harness import cli + from fi.alk.harness.sources import resolve + from fi.alk.harness.understand import opening + + source = _repo(tmp_path) + fake = model() + raw = _job(metadata={"generic_harness_v1": True}) + + _run(tmp_path, raw) + + job = HarnessJob.model_validate(raw) + built = opening(resolve("repo", name=source.name, root=source)) + cli._guidance( + argparse.Namespace(guidance=job.authoring_guidance()) + ) + assert fake.sent[0] == built + assert "standalone Dockerfile cannot be built" in fake.sent[0] + + +def test_a_generic_jobs_reading_refuses_what_the_builds_gate_refuses(tmp_path, model): + # The build's gate for a generic job: a chat agent must declare where it keeps its state. + refused = _checked_contract( + dict(CHAT), source_root=None, require_setup_contract=True, subfolder=None + ) + assert isinstance(refused, dict) + assert "data_store" in refused["content"][0]["text"] + _repo(tmp_path) + fake = model(_submits(CHAT)) + output = tmp_path / "reading" + + assert _run(tmp_path, _job(metadata={"generic_harness_v1": True})) == 78 + + assert fake.submissions == 3 + assert not (output / "contract.json").exists() + assert _failure_in(output)["code"] == "agent_reading_incomplete" + + +def test_the_claude_backend_accepts_the_reading_spec(tmp_path, model, monkeypatch): + from fi.alk.harness.backends.claude import ClaudeBackend + + monkeypatch.setenv("ALK_CLAUDE_GATEWAY_URL", "http://127.0.0.1:9") + monkeypatch.setenv("ALK_CLAUDE_GATEWAY_API_KEY", "placeholder-not-a-key") + _repo(tmp_path) + fake = model() + _run(tmp_path) + [spec] = fake.created + + options = ClaudeBackend().create(spec)._options + try: + assert options.max_turns == 70 + assert getattr(options, "thinking", None) is not None + assert any(name.endswith("submit_contract") for name in options.allowed_tools) + finally: + prompt = options.system_prompt + if isinstance(prompt, dict) and prompt.get("path"): + Path(prompt["path"]).unlink(missing_ok=True) + + +def test_the_soft_limit_counts_from_the_readings_start(tmp_path, model, monkeypatch): + _repo(tmp_path) + _short_limit(monkeypatch, soft=30.0) + real = agent_reading.open_stage + now = [1000.0] + # The reading's own clock, so the time spent opening the stage is told, not slept. + monkeypatch.setattr(agent_reading, "time", types.SimpleNamespace(monotonic=lambda: now[0])) + + def slow(*args, **kwargs): + now[0] += 29.9 + return real(*args, **kwargs) + + monkeypatch.setattr(agent_reading, "open_stage", slow) + fake = model(_quiet) + started = time.monotonic() + + assert _run(tmp_path, wait=0.2) == 78 + + # 0.1 s were left of the 30, not 30 again from the stage's opening. + [interrupted] = fake.interrupts + assert interrupted - started < 10.0 + + +# -- R37 to R41: the switch, the folders, the limit and the last write -------------------------- + + +def _never(*_args, **_kwargs): + raise AssertionError("a reading must not reach the build's setup") + + +def test_the_switch_runs_only_the_reading(tmp_path, model, monkeypatch): + _repo(tmp_path) + monkeypatch.setattr(authoring_entrypoint, "initialize_hosted_compose", _never) + monkeypatch.setattr(authoring_entrypoint, "_load_provider_import_profile", _never) + model(_writes(CHAT)) + job_path = tmp_path / "job.json" + job_path.write_text(json.dumps(_job()), encoding="utf-8") + output = tmp_path / "reading" + + code = authoring_entrypoint.main( + [str(job_path), "--source", str(tmp_path / "source"), "--output", str(output), "--read-agent-only"] + ) + + assert code == 0 + assert (output / "agent.json").exists() + + +def test_the_switch_sends_the_readings_start_to_stderr(tmp_path, model, monkeypatch, capfd): + _repo(tmp_path) + # As in the sandbox: nothing has set up logging before the switch runs. + monkeypatch.setattr(logging.getLogger(), "handlers", []) + monkeypatch.setattr(reading_entry.logger, "handlers", []) + monkeypatch.setattr(reading_entry.logger, "level", logging.NOTSET) + model(_writes(CHAT)) + job_path = tmp_path / "job.json" + job_path.write_text(json.dumps(_job()), encoding="utf-8") + output = tmp_path / "reading" + + code = authoring_entrypoint.main( + [str(job_path), "--source", str(tmp_path / "source"), "--output", str(output), "--read-agent-only"] + ) + + assert code == 0 + assert "reading started at " in capfd.readouterr().err + + +def test_a_bad_job_through_main_is_refused(tmp_path, model): + fake = model() + job_path = tmp_path / "job.json" + job_path.write_text("{not a job", encoding="utf-8") + output = tmp_path / "reading" + + code = authoring_entrypoint.main( + [str(job_path), "--source", str(tmp_path / "source"), "--output", str(output), "--read-agent-only"] + ) + + assert code == 78 + assert _failure_in(output)["code"] == READING_JOB_INVALID + assert fake.created == [] + + +def _everything_under(root: Path) -> dict[str, bytes]: + return { + path.relative_to(root).as_posix(): path.read_bytes() + for path in sorted(root.rglob("*")) + if path.is_file() + } + + +@pytest.mark.parametrize("ends", ["succeeds", "fails"]) +def test_nothing_is_written_outside_the_output(tmp_path, model, ends): + source = _repo(tmp_path) + work = tmp_path / "work" + (work / "authoring").mkdir(parents=True) + (work / "authoring" / "kept.txt").write_text("the build's own folder", encoding="utf-8") + (work / "job.json").write_text(json.dumps(_job(subfolder=SUPPORT)), encoding="utf-8") + output = work / "reading" + # The real gate, so the contract lands wherever the reading's own stage writes it. + model(_submits(CHAT) if ends == "succeeds" else _ends(), output=output) + before = _everything_under(tmp_path) + + code = run_reading_mode( + agent_reading.READ_AGENT, job_path=work / "job.json", source=source, output=output + ) + + assert code == (0 if ends == "succeeds" else 78) + assert (output / "agent.json").exists() is (ends == "succeeds") + after = { + name: body + for name, body in _everything_under(tmp_path).items() + if not name.startswith("work/reading/") + } + assert after == before + + +@pytest.mark.parametrize("value", ["abc", "0", "-5", "", "nan", "inf"]) +def test_a_bad_time_limit_falls_back_to_the_default(monkeypatch, value): + monkeypatch.setenv(_LIMIT_ENV, value) + + assert agent_reading._soft_limit() == 1800.0 + assert reading_entry._hard_limit(agent_reading.READ_AGENT) == 1800.0 + 300.0 + + +def test_a_good_time_limit_is_read(monkeypatch): + monkeypatch.setenv(_LIMIT_ENV, "120") + + assert agent_reading._soft_limit() == 120.0 + mode = agent_reading.READ_AGENT + # The helper's limit is the reading's own plus its margin, and a timeout repeats if retried. + assert reading_entry._hard_limit(mode) == 120.0 + 300.0 + assert mode.timed_out_retryable is False + # The contract's codes, as the platform reads them. + assert (mode.timed_out_code, mode.crashed_code) == ( + "agent_reading_timed_out", + "agent_reading_crashed", + ) + + +def test_a_fault_after_a_saved_contract_still_leaves_the_document(tmp_path, model, caplog): + _repo(tmp_path) + fake = model() + # The contract lands, then the session fails to open: the fault the reading raises. + fake.saves_on_start = CHAT + fake.start_raises = True + output = tmp_path / "reading" + + with caplog.at_level("WARNING", logger=agent_reading.__name__): + assert _run(tmp_path) == 0 + + assert _names(output) == {"contract.json", "agent.json", "cost.json"} + assert fake.sent == [] + _assert_cost(output, 0.0, False) + # The line names the class; the trace, masked like every line here, keeps the rest. + [logged] = [ + record + for record in caplog.records + if record.name == agent_reading.__name__ and "saved contract" in record.getMessage() + ] + assert "FileNotFoundError" in logged.getMessage() + assert "was not found" not in logged.getMessage() + assert "FileNotFoundError: the Claude CLI was not found" in logged.exc_text + + +def test_a_session_that_cannot_open_is_a_crash(tmp_path, model): + _repo(tmp_path) + fake = model() + fake.start_raises = True + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_crashed", True) + assert "FileNotFoundError" in failure["message"] + assert fake.sent == [] + _assert_cost(output, 0.0, False) + + +def test_a_failed_document_write_is_not_retried(tmp_path, model, monkeypatch): + _repo(tmp_path) + real = os.replace + + def refuses_the_document(source, destination, *args, **kwargs): + if Path(destination).name == "agent.json": + raise OSError("no space left on the device") + return real(source, destination, *args, **kwargs) + + monkeypatch.setattr(os, "replace", refuses_the_document) + model(_writes(CHAT, cost_usd=0.1)) + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_document_invalid", False) + # No document, and no half-written one left beside it. + assert _names(output) == {"contract.json", "cost.json", "failure.json"} + _assert_cost(output, 0.1, True) diff --git a/tests/harness/test_agent_scan.py b/tests/harness/test_agent_scan.py new file mode 100644 index 000000000..4c2c422ac --- /dev/null +++ b/tests/harness/test_agent_scan.py @@ -0,0 +1,1222 @@ +"""The find-agents mode: what it refuses, the file it leaves on every path, and the cost it leaves.""" + +from __future__ import annotations + +import asyncio +import inspect +import json +import logging +import math +import os +import shutil +import time +from pathlib import Path + +import pytest + +from fi.alk.harness import agent_scan, reading_entry, scan_walk, spend +from fi.alk.harness.agent_scan import FIND_AGENTS, _check_job, _hints, _work +from fi.alk.harness.job import ( + FailureDomain, + HarnessJob, + HarnessStage, + RepositorySource, + SourceKind, +) +from fi.alk.harness.scan_ai import AiOutcome +from fi.alk.harness.scan_document import ScanResultInvalid, wire_problems +from fi.alk.harness.scan_folders import ClockRanOut, FolderCheckerBroken, FolderWord +from fi.alk.harness.scan_walk import Candidate, WalkResult + +GRAPH = "from langgraph.graph import StateGraph\nStateGraph(dict)\n" +SCRIPT_AGENT = ( + "import { createReactAgent } from '@langchain/langgraph/prebuilt';\n" + "export const agent = createReactAgent({ llm, tools: [] });\n" +) +MODEL_HELPER = ( + "from langchain_openai import ChatOpenAI\n" + "def judge(x):\n" + " return ChatOpenAI().invoke(x)\n" +) +# A TypeScript agent the walk can't list, beside a Python file that only calls a model. +SCRIPT_BESIDE_A_HELPER = {"web/src/agent.ts": SCRIPT_AGENT, "backend/judge.py": MODEL_HELPER} +CLIENT = "from openai import OpenAI\nOpenAI()\n" +# Both kinds of file that only call a model: a chat model, and a provider's own client. +HELPERS = {"chat_model": MODEL_HELPER, "client": CLIENT} +PLAIN = "def main():\n return 1\n" +COMMIT = "AB" * 20 +# Plain code outside the side folders and one client inside them: the walk lists only a side +# agent, so the model is asked. +SIDE_ONLY = {"app/main.py": PLAIN, "tests/test_bot.py": CLIENT} +AI_SENTENCE = "Found by the AI step, not by the code check." + + +class FakeChecker: + """Answers every folder at once. From call ``fail_at`` on (counted from 1) it raises ``error``.""" + + def __init__(self) -> None: + self.name = "langgraph" + self.fail_at: int | None = None + self.error: Exception = ClockRanOut("the clock ran out") + self.calls: list[str] = [] + + def __enter__(self) -> FakeChecker: + return self + + def __exit__(self, *exc: object) -> None: + return None + + def check(self, source, subfolder, *, deadline, clock=None) -> FolderWord: + self.calls.append(subfolder) + if self.fail_at is not None and len(self.calls) >= self.fail_at: + raise self.error + return FolderWord({"name": self.name, "version": None, "label": None}, finished=True) + + +class FakeAi: + """The model step: one turn ends, then ``during`` runs, then ``outcome`` comes back.""" + + def __init__(self) -> None: + self.outcome = _outcome("empty") + self.during = None + self.calls: list[dict] = [] + + async def __call__(self, source, *, hints, seconds, on_turn_end) -> AiOutcome: + self.calls.append({"source": source, "hints": hints, "seconds": seconds}) + on_turn_end() + if self.during is not None: + self.during() + return self.outcome + + +def _outcome( + kind, pairs=(), *, refused=0, complete=True, retryable=False, uncarried=0 +) -> AiOutcome: + return AiOutcome(kind, tuple(pairs), refused, complete, retryable, "fixed words", uncarried) + + +@pytest.fixture(autouse=True) +def _fresh_state(monkeypatch, tmp_path): + monkeypatch.delenv(spend.JOURNAL_ALIAS, raising=False) + monkeypatch.setattr(spend, "_path", None) + # The machine's own secrets files must not decide what a test sees masked. + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (tmp_path / "no-secrets.json",)) + monkeypatch.setattr(agent_scan, "CLOCK", 1200.0) + monkeypatch.setattr(agent_scan, "AI_SECONDS", 600.0) + # The scan reads these when it starts, so the machine's own settings mustn't decide a test. + monkeypatch.delenv(agent_scan.CLOCK_ENV, raising=False) + monkeypatch.delenv(agent_scan.AI_ENV, raising=False) + spend._stages.clear() + yield + spend._stages.clear() + + +@pytest.fixture(autouse=True) +def checker(monkeypatch) -> FakeChecker: + fake = FakeChecker() + monkeypatch.setattr(agent_scan, "FolderChecker", lambda: fake) + return fake + + +@pytest.fixture(autouse=True) +def ai(monkeypatch) -> FakeAi: + fake = FakeAi() + monkeypatch.setattr(agent_scan, "run_ai_step", fake) + return fake + + +def _write(folder: Path, files: dict[str, str]) -> Path: + for relative, content in files.items(): + path = folder / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return folder + + +def _job(kind: str = "github", **source: object) -> HarnessJob: + return HarnessJob.model_validate(_job_data(kind, **source)) + + +def _job_data(kind: str = "github", **source: object) -> dict: + sources = { + "github": { + "kind": "github", + "visibility": "public", + "repository": "acme/support-bot", + "commit_sha": COMMIT, + }, + "archive": {"kind": "archive", "archive_artifact_id": "upload-1"}, + } + return { + "job_id": "job", + "run_id": "run", + "execution": "hosted", + "source": {**sources[kind], **source}, + "runtime": {"isolation": "dedicated_vm"}, + "agent": {"connector": "auto"}, + } + + +def _scan(job: HarnessJob, source: Path, output: Path) -> int: + return asyncio.run(_work(job, source, output)) + + +def _read_mode(kind: str, source: Path, output: Path) -> int: + """The whole mode, through the helper that runs every reading, from a job file.""" + job_path = output.parent / f"job-{output.name}.json" + job_path.write_text(json.dumps(_job_data(kind)), encoding="utf-8") + return reading_entry.run_reading_mode( + FIND_AGENTS, job_path=job_path, source=source, output=output + ) + + +def _read(output: Path, name: str) -> dict: + return json.loads((output / name).read_text(encoding="utf-8")) + + +def _agents(output: Path) -> dict: + document = _read(output, "agents.json") + assert wire_problems(document) == [] + if document["error"] is not None: + assert document["error"]["message"].strip() + return document + + +def _entries(document: dict) -> list[str]: + return [agent["entry"] for agent in document["agents"]] + + +def _candidate(entry: str, subfolder: str, *, side: bool = True) -> Candidate: + return Candidate( + entry=entry, + subfolder=subfolder, + sign="client", + side=side, + builds=(), + build_may_stop="", + partial=False, + env_names=(), + tools=0, + prompt_files=0, + evidence=(), + ) + + +def _walk(**changes: object) -> WalkResult: + values: dict = { + "agents": (), + "more_than_fifty": False, + "stopped": "", + "unread_files": 0, + "source_outside_side": True, + "candidate_outside_side": False, + "script_outside_side": False, + "only_weak_outside_side": False, + "paths_left_out": 0, + "upload_others": 0, + "upload_others_side": 0, + } + values.update(changes) + return WalkResult(**values) + + +def _tree(folder: Path, leave_out: Path) -> dict[str, tuple[int, int]]: + seen = {} + for directory, directories, files in os.walk(folder, followlinks=False): + for name in directories + files: + path = Path(directory, name) + if path == leave_out or leave_out in path.parents: + continue + status = os.lstat(path) + seen[str(path)] = (status.st_size, status.st_mtime_ns) + return seen + + +def test_a_remote_source_or_a_subfolder_is_refused() -> None: + github = _job() + jobs = [ + github.model_copy( + update={ + "source": RepositorySource.model_construct( + kind=SourceKind.REMOTE, endpoint="https://agent.example" + ) + } + ), + github.model_copy( + update={"source": RepositorySource.model_construct(kind=SourceKind.PROVIDER)} + ), + _job(subfolder="services/bot"), + ] + + for job in jobs: + refused = _check_job(job) + assert refused is not None + assert refused.code == "scan_job_invalid" + assert refused.domain is FailureDomain.PLATFORM_SYNC + assert refused.stage is HarnessStage.UNDERSTANDING_AGENT + assert refused.retryable is False + assert refused.message.strip() + + +def test_the_scan_job_passes_the_kits_job_check() -> None: + # Built through HarnessJob's own validation, as the helper parses the job. + for job in (_job(), _job("archive"), _job(subfolder="")): + assert _check_job(job) is None + + +def test_the_mode_settings(monkeypatch) -> None: + assert FIND_AGENTS.name == "find_agents" + assert FIND_AGENTS.check_job is _check_job + assert FIND_AGENTS.work is _work + assert inspect.iscoroutinefunction(FIND_AGENTS.work) + # The helper reads the scan's own clock variable for its hard stop, as the scan does. + assert FIND_AGENTS.time_limit_seconds == agent_scan.CLOCK == 1200.0 + assert FIND_AGENTS.limit_env == agent_scan.CLOCK_ENV == "ALK_SCAN_MAX_SECONDS" + assert FIND_AGENTS.limit_margin_seconds == 600.0 + assert FIND_AGENTS.timed_out_code == "scan_hard_limit_reached" + assert FIND_AGENTS.timed_out_retryable is False + assert FIND_AGENTS.crashed_code == "scan_crashed" + + +def test_a_missing_or_empty_source_is_a_retryable_failure(tmp_path, ai) -> None: + a_file = _write(tmp_path / "files", {"bot.py": GRAPH}) / "bot.py" + empty = tmp_path / "empty" + (empty / "sub").mkdir(parents=True) + linked = tmp_path / "linked" + linked.mkdir() + (linked / "bot.py").symlink_to(a_file) + + for index, source in enumerate((tmp_path / "missing", a_file, empty, linked)): + output = tmp_path / f"reading-{index}" + + assert _scan(_job(), source, output) == 1 + + failure = _read(output, "failure.json") + assert failure["code"] == "scan_source_missing" + assert failure["domain"] == "platform_sync" + assert failure["stage"] == "understanding_agent" + assert failure["retryable"] is True + assert not (output / "agents.json").exists() + assert ai.calls == [] + + +def test_none_when_nothing_looks_like_an_agent(tmp_path, ai) -> None: + quiet = _write(tmp_path / "quiet", {"README.md": "# notes\n"}) + + assert _scan(_job(), quiet, tmp_path / "out-quiet") == 0 + + document = _agents(tmp_path / "out-quiet") + assert (document["status"], document["agents"], document["truncated"]) == ("none", [], False) + assert ai.calls == [] + + plain = _write(tmp_path / "plain", {"app/main.py": PLAIN}) + ai.outcome = _outcome("empty") + + assert _scan(_job(), plain, tmp_path / "out-plain") == 0 + + document = _agents(tmp_path / "out-plain") + assert (document["status"], document["agents"]) == ("none", []) + assert len(ai.calls) == 1 + + +def test_the_walk_limit_before_any_agent_is_an_error(tmp_path, monkeypatch, ai) -> None: + source = _write(tmp_path / "source", {"app/main.py": PLAIN}) + monkeypatch.setattr( + agent_scan, "walk_repo", lambda *args, **kwargs: _walk(stopped="limit", unread_files=2) + ) + + assert _scan(_job(), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert document["status"] == "error" + assert document["error"]["code"] == "scan_limit_reached" + assert document["truncated"] is True + assert ai.calls == [] + + +def test_the_clock_ends_in_a_file(tmp_path, monkeypatch, ai) -> None: + source = _write(tmp_path / "source", {"a/graph.py": GRAPH}) + monkeypatch.setattr(agent_scan, "CLOCK", -1.0) + + assert _scan(_job(), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert document["status"] == "error" + assert document["error"]["code"] == "scan_timed_out" + assert ai.calls == [] + + +def test_a_clock_out_during_the_checks_lists_the_agents_unchecked(tmp_path, checker) -> None: + source = _write(tmp_path / "source", {"a/graph.py": GRAPH, "b/graph.py": GRAPH}) + checker.fail_at = 1 + + assert _scan(_job(), source, tmp_path / "first") == 0 + + document = _agents(tmp_path / "first") + assert (document["status"], _entries(document)) == ("found", ["a/graph.py", "b/graph.py"]) + assert document["truncated"] is True + assert {agent["framework"]["name"] for agent in document["agents"]} == {"unknown"} + assert {agent["confidence"] for agent in document["agents"]} == {"low"} + + checker.calls.clear() + checker.fail_at = 2 + + assert _scan(_job(), source, tmp_path / "second") == 0 + + document = _agents(tmp_path / "second") + assert (document["status"], _entries(document)) == ("found", ["a/graph.py", "b/graph.py"]) + assert document["agents"][1]["framework"]["name"] == "unknown" + assert document["truncated"] is True + + +def test_an_upload_gives_one_agent_at_the_root(tmp_path) -> None: + source = _write(tmp_path / "source", {"a/graph.py": GRAPH, "b/graph.py": GRAPH}) + + assert _scan(_job("archive", commit_sha=COMMIT), source, tmp_path / "upload") == 0 + + document = _agents(tmp_path / "upload") + assert document["status"] == "found" + assert [agent["subfolder"] for agent in document["agents"]] == [""] + assert document["truncated"] is True + assert document["commit_sha"] is None + + assert _scan(_job(), source, tmp_path / "github") == 0 + + document = _agents(tmp_path / "github") + assert sorted(_entries(document)) == ["a/graph.py", "b/graph.py"] + assert document["commit_sha"] == COMMIT.lower() + + +def test_an_uploads_ai_row_is_at_the_root(tmp_path, ai) -> None: + source = _write(tmp_path / "source", {"svc/main.py": PLAIN, "svc/bot.py": PLAIN}) + ai.outcome = _outcome("pairs", [("svc", "svc/main.py"), ("svc", "svc/bot.py")]) + + assert _scan(_job("archive"), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert len(ai.calls) == 1 + assert _entries(document) == ["svc/main.py"] + assert document["agents"][0]["subfolder"] == "" + assert document["truncated"] is True + assert document["commit_sha"] is None + + +def test_the_ai_runs_unless_an_agent_is_found_outside_side_folders(tmp_path, monkeypatch, ai) -> None: + """Agents only in test or example folders still ask the model, given the time; an agent + outside them doesn't. The weak-sign-beside-TS/JS case has its own tests below.""" + side_only = _write(tmp_path / "side-only", SIDE_ONLY) + + monkeypatch.setattr(agent_scan, "CLOCK", 1000.0) + assert _scan(_job(), side_only, tmp_path / "out-1000") == 0 + assert [call["seconds"] for call in ai.calls] == [600.0] + assert "test_bot" not in ai.calls[0]["hints"] + + ai.calls.clear() + monkeypatch.setattr(agent_scan, "CLOCK", 500.0) + assert _scan(_job(), side_only, tmp_path / "out-500") == 0 + assert len(ai.calls) == 1 + # What's left after the walk, less the 180 s kept for what follows the model. + assert 300.0 < ai.calls[0]["seconds"] <= 320.0 + + ai.calls.clear() + monkeypatch.setattr(agent_scan, "CLOCK", 300.0) + assert _scan(_job(), side_only, tmp_path / "out-300") == 0 + assert ai.calls == [] + + monkeypatch.setattr(agent_scan, "CLOCK", 1200.0) + found = _write(tmp_path / "found", {**SIDE_ONLY, "bot/graph.py": GRAPH}) + assert _scan(_job(), found, tmp_path / "out-found") == 0 + assert ai.calls == [] + assert "bot/graph.py" in _entries(_agents(tmp_path / "out-found")) + + only_side = _write(tmp_path / "only-side", {"tests/test_bot.py": CLIENT}) + assert _scan(_job(), only_side, tmp_path / "out-only-side") == 0 + assert ai.calls == [] + + +def test_hints_carry_no_path(tmp_path, monkeypatch, ai) -> None: + walk = _walk( + agents=(_candidate("examples/zebra_place/quokka_bot.py", "examples/zebra_place"),), + unread_files=3, + ) + + hints = _hints(walk) + + for word in ("zebra", "quokka", "examples", "/", ".py"): + assert word not in hints + assert "1" in hints and "3" in hints + + source = _write(tmp_path / "source", {"app/main.py": PLAIN}) + monkeypatch.setattr(agent_scan, "walk_repo", lambda *args, **kwargs: walk) + + assert _scan(_job(), source, tmp_path / "out") == 0 + + assert ai.calls[0]["hints"] == hints + + +def test_side_rows_and_ai_rows_together_keep_fifty(tmp_path, checker, ai) -> None: + files = {f"tests/test_bot_{index:02d}.py": CLIENT for index in range(30)} + files.update({f"ai/bot_{index:02d}.py": PLAIN for index in range(30)}) + source = _write(tmp_path / "source", files) + # The last pair repeats a walk entry under a different folder. + ai.outcome = _outcome( + "pairs", + [("ai", f"ai/bot_{index:02d}.py") for index in range(30)] + [("", "tests/test_bot_00.py")], + ) + + assert _scan(_job(), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert len(document["agents"]) == 50 + assert document["truncated"] is True + repeated = [agent for agent in document["agents"] if agent["entry"] == "tests/test_bot_00.py"] + assert [agent["subfolder"] for agent in repeated] == ["tests"] + assert AI_SENTENCE not in repeated[0]["evidence"] + # 30 walk rows and 30 new AI rows; the repeated entry is never checked. + assert len(checker.calls) == 60 + + +def test_a_walk_row_beats_an_ai_row_for_the_same_entry(tmp_path, checker, ai) -> None: + source = _write(tmp_path / "source", SIDE_ONLY) + ai.outcome = _outcome("pairs", [("", "tests/test_bot.py")]) + + assert _scan(_job(), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert len(ai.calls) == 1 + assert _entries(document) == ["tests/test_bot.py"] + assert document["agents"][0]["subfolder"] == "tests" + assert AI_SENTENCE not in document["agents"][0]["evidence"] + assert checker.calls == ["tests"] + + +def test_a_crash_after_the_ai_step_keeps_what_was_paid_for( + tmp_path, monkeypatch, checker, ai +) -> None: + source = _write(tmp_path / "source", SIDE_ONLY) + ai.outcome = _outcome("pairs", [("app", "app/main.py")]) + checker.fail_at = 2 + checker.error = FolderCheckerBroken("the child ended") + + assert _scan(_job(), source, tmp_path / "broken-check") == 0 + + document = _agents(tmp_path / "broken-check") + assert document["status"] == "found" + assert sorted(_entries(document)) == ["app/main.py", "tests/test_bot.py"] + assert document["truncated"] is True + paid = next(agent for agent in document["agents"] if agent["entry"] == "app/main.py") + assert paid["framework"]["name"] == "unknown" + assert not (tmp_path / "broken-check" / "failure.json").exists() + + checker.fail_at, checker.calls = None, [] + + def out_of_memory(*args, **kwargs): + raise MemoryError() + + with monkeypatch.context() as patched: + patched.setattr(agent_scan, "describe_entry", out_of_memory) + + assert _scan(_job(), source, tmp_path / "describe") == 0 + + document = _agents(tmp_path / "describe") + assert (document["status"], _entries(document)) == ("found", ["tests/test_bot.py"]) + assert document["truncated"] is True + + real_write = agent_scan.write_agents_json + + def refuse_found(output, body): + if b'"status":"found"' in body: + raise OSError("disk full") + return real_write(output, body) + + checker.fail_at, checker.calls = 2, [] + with monkeypatch.context() as patched: + patched.setattr(agent_scan, "write_agents_json", refuse_found) + + assert _scan(_job(), source, tmp_path / "write-fails") == 0 + + document = _agents(tmp_path / "write-fails") + assert document["status"] == "error" + assert document["error"]["code"] == "scan_failed_after_ai" + assert document["agents"] == [] + assert not (tmp_path / "write-fails" / "failure.json").exists() + + def refuse_all(output, body): + raise OSError("disk full") + + checker.calls = [] + with monkeypatch.context() as patched: + patched.setattr(agent_scan, "write_agents_json", refuse_all) + + with pytest.raises(FolderCheckerBroken): + _scan(_job(), source, tmp_path / "nothing-writes") + + assert (tmp_path / "nothing-writes" / "cost.json").exists() + + +def test_an_upload_prefers_a_non_side_ai_row_to_a_side_walk_row(tmp_path, ai) -> None: + source = _write(tmp_path / "source", SIDE_ONLY) + ai.outcome = _outcome("pairs", [("app", "app/main.py")]) + + assert _scan(_job("archive"), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert _entries(document) == ["app/main.py"] + assert document["agents"][0]["subfolder"] == "" + assert document["truncated"] is True + + +@pytest.mark.parametrize("retryable", [False, True], ids=["refused", "retryable"]) +def test_a_model_error_with_walk_rows_keeps_them(tmp_path, ai, retryable) -> None: + ai.outcome = _outcome("model_error", retryable=retryable) + with_rows = _write(tmp_path / "with-rows", SIDE_ONLY) + + assert _scan(_job(), with_rows, tmp_path / "out-rows") == 0 + + document = _agents(tmp_path / "out-rows") + assert (document["status"], _entries(document)) == ("found", ["tests/test_bot.py"]) + assert document["truncated"] is True + assert not (tmp_path / "out-rows" / "failure.json").exists() + + without = _write(tmp_path / "without", {"app/main.py": PLAIN}) + + assert _scan(_job(), without, tmp_path / "out-none") == (1 if retryable else 78) + + failure = _read(tmp_path / "out-none", "failure.json") + assert (failure["code"], failure["retryable"]) == ("scan_model_error", retryable) + assert not (tmp_path / "out-none" / "agents.json").exists() + + +def test_an_ai_row_the_clock_leaves_unchecked_is_still_written(tmp_path, checker, ai) -> None: + source = _write(tmp_path / "source", SIDE_ONLY) + ai.outcome = _outcome("pairs", [("app", "app/main.py")]) + checker.fail_at = 2 + + assert _scan(_job(), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + paid = next(agent for agent in document["agents"] if agent["entry"] == "app/main.py") + assert paid["framework"]["name"] == "unknown" + assert paid["confidence"] == "low" + assert document["truncated"] is True + + +def test_cost_is_written_on_every_return_with_complete(tmp_path, monkeypatch, ai) -> None: + # Driven without the helper, so only the work's own `finally` can have written these. + assert _scan(_job(), tmp_path / "missing", tmp_path / "missing-out") == 1 + assert _read(tmp_path / "missing-out", "cost.json")["complete"] is True + + found = _write(tmp_path / "found", {"a/graph.py": GRAPH}) + assert _scan(_job(), found, tmp_path / "found-out") == 0 + assert _read(tmp_path / "found-out", "cost.json")["complete"] is True + + side_only = _write(tmp_path / "side-only", SIDE_ONLY) + during: list[bool] = [] + ai.during = lambda: during.append(_read(tmp_path / "turn-out", "cost.json")["complete"]) + ai.outcome = _outcome("pairs", [("app", "app/main.py")], complete=False) + + assert _scan(_job(), side_only, tmp_path / "turn-out") == 0 + + assert during == [False] + assert _read(tmp_path / "turn-out", "cost.json")["complete"] is False + + ai.during = None + ai.outcome = _outcome("forbidden") + assert _scan(_job(), side_only, tmp_path / "forbidden-out") == 78 + assert _read(tmp_path / "forbidden-out", "cost.json")["complete"] is True + + def crash(*args, **kwargs): + raise RuntimeError("walk broke") + + monkeypatch.setattr(agent_scan, "walk_repo", crash) + with pytest.raises(RuntimeError): + _scan(_job(), found, tmp_path / "crash-out") + assert (tmp_path / "crash-out" / "cost.json").exists() + + +def test_a_memory_error_ends_in_a_file(tmp_path, monkeypatch, checker, ai) -> None: + source = _write(tmp_path / "source", SIDE_ONLY) + + def out_of_memory(*args, **kwargs): + raise MemoryError() + + monkeypatch.setattr(agent_scan, "walk_repo", out_of_memory) + + assert _scan(_job(), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert document["status"] == "error" + assert document["error"]["code"] == "scan_out_of_memory" + assert checker.calls == [] + assert ai.calls == [] + + +def test_a_broken_checker_before_the_ai_is_a_crash(tmp_path, checker, ai) -> None: + checker.fail_at = 1 + checker.error = FolderCheckerBroken("the child could not start") + + for name, files in (("found", {"a/graph.py": GRAPH}), ("side-only", SIDE_ONLY)): + output = tmp_path / f"out-{name}" + + assert _scan(_job(), _write(tmp_path / name, files), output) == 1 + + failure = _read(output, "failure.json") + assert (failure["code"], failure["domain"], failure["retryable"]) == ( + "scan_crashed", + "infrastructure", + True, + ) + assert not (output / "agents.json").exists() + assert ai.calls == [] + + +@pytest.mark.parametrize( + ("kind", "retryable", "code", "written"), + [ + ("model_error", True, 1, ("failure", "scan_model_error", True)), + ("model_error", False, 78, ("failure", "scan_model_error", False)), + ("forbidden", False, 78, ("failure", "scan_model_forbidden", False)), + ("no_answer", False, 0, ("error", "scan_ai_no_answer")), + ("timed_out", False, 0, ("error", "scan_timed_out")), + ("empty", False, 0, ("none",)), + ], +) +def test_each_ai_outcome_gives_its_exit_code(tmp_path, ai, kind, retryable, code, written) -> None: + source = _write(tmp_path / "source", {"app/main.py": PLAIN}) + ai.outcome = _outcome(kind, retryable=retryable) + output = tmp_path / "out" + + assert _scan(_job(), source, output) == code + + assert len(ai.calls) == 1 + if written[0] == "failure": + failure = _read(output, "failure.json") + assert (failure["code"], failure["retryable"]) == written[1:] + assert failure["domain"] == "infrastructure" + assert not (output / "agents.json").exists() + else: + document = _agents(output) + assert document["status"] == written[0] + if written[0] == "error": + assert document["error"]["code"] == written[1] + assert not (output / "failure.json").exists() + + +def test_every_failure_goes_through_the_helper(tmp_path, monkeypatch, checker, ai) -> None: + ran = tmp_path / "ran" + source = _write( + tmp_path / "source", + {**SIDE_ONLY, "app/main.py": PLAIN + f"open({str(ran)!r}, 'w').close()\n"}, + ) + plain = _write(tmp_path / "plain", {"app/main.py": PLAIN}) + written = [] + real = agent_scan.write_reading_failure + + def recording(failure, output): + written.append(failure) + return real(failure, output) + + monkeypatch.setattr(agent_scan, "write_reading_failure", recording) + + def invalid(**kwargs): + raise ScanResultInvalid("agents.too_many") + + def broken_checker(): + checker.fail_at, checker.error = 1, FolderCheckerBroken("no child") + + def bad_document(patched): + patched.setattr(agent_scan, "finish", invalid) + + def bad_document_after_ai(patched): + ai.outcome = _outcome("pairs", [("app", "app/main.py")]) + patched.setattr(agent_scan, "finish", invalid) + + cases = [ + ("missing", tmp_path / "missing", lambda patched: None, 1, "scan_source_missing"), + ("broken", source, lambda patched: broken_checker(), 1, "scan_crashed"), + ( + "retryable", + plain, + lambda patched: setattr(ai, "outcome", _outcome("model_error", retryable=True)), + 1, + "scan_model_error", + ), + ( + "refused", + plain, + lambda patched: setattr(ai, "outcome", _outcome("model_error")), + 78, + "scan_model_error", + ), + ( + "forbidden", + plain, + lambda patched: setattr(ai, "outcome", _outcome("forbidden")), + 78, + "scan_model_forbidden", + ), + ("invalid", source, bad_document, 78, "scan_result_invalid"), + ("invalid-after-ai", source, bad_document_after_ai, 78, "scan_result_invalid"), + ] + for name, scanned, arrange, code, failure_code in cases: + checker.fail_at, checker.calls = None, [] + ai.outcome = _outcome("empty") + output = tmp_path / f"out-{name}" + before = _tree(tmp_path, output) + with monkeypatch.context() as patched: + arrange(patched) + + assert _scan(_job(), scanned, output) == code + + assert written[-1].code == failure_code + assert written[-1].stage is HarnessStage.UNDERSTANDING_AGENT + assert written[-1].message.strip() + failure = _read(output, "failure.json") + assert (failure["code"], failure["stage"], failure["message"]) == ( + failure_code, + "understanding_agent", + written[-1].message, + ) + assert not (output / "agents.json").exists() + # Nothing is written outside the output folder, and the source never runs. + assert _tree(tmp_path, output) == before + + # A scan that ends in agents.json writes it into the output folder and nowhere else. + checker.fail_at, checker.calls = None, [] + ai.outcome = _outcome("empty") + output = tmp_path / "out-found" + before = _tree(tmp_path, output) + + assert _scan(_job(), source, output) == 0 + + assert _agents(output)["status"] == "found" + assert _tree(tmp_path, output) == before + assert len(written) == len(cases) + assert not ran.exists() + + +def test_a_refused_pair_in_the_kept_submit_sets_truncated(tmp_path, ai) -> None: + source = _write(tmp_path / "source", {"app/main.py": PLAIN}) + + for refused, truncated in ((0, False), (1, True)): + ai.outcome = _outcome("pairs", [("app", "app/main.py")], refused=refused) + output = tmp_path / f"out-{refused}" + + assert _scan(_job(), source, output) == 0 + + document = _agents(output) + assert _entries(document) == ["app/main.py"] + assert document["truncated"] is truncated + + +def test_a_crash_after_the_ai_step_logs_its_trace_masked(tmp_path, monkeypatch, caplog, ai) -> None: + source = _write(tmp_path / "source", SIDE_ONLY) + ai.outcome = _outcome("pairs", [("app", "app/main.py")]) + secret = "FAKE_SECRET_VALUE_123" + secrets = tmp_path / "secrets.json" + secrets.write_text(json.dumps({"SIMULATOR_OPENAI_API_KEY": secret}), encoding="utf-8") + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (secrets,)) + + def crash(*args, **kwargs): + raise RuntimeError(f"bad value {secret}") + + monkeypatch.setattr(agent_scan, "describe_entry", crash) + with caplog.at_level(logging.DEBUG, logger=agent_scan.logger.name): + assert _scan(_job(), source, tmp_path / "out") == 0 + + assert "failed after the model step" in caplog.text + assert "Traceback" in caplog.text and "RuntimeError" in caplog.text + assert secret not in caplog.text + assert _agents(tmp_path / "out")["status"] == "found" + + +def test_an_uploads_crash_after_the_ai_step_still_writes_one_row(tmp_path, checker, ai) -> None: + source = _write(tmp_path / "source", {**SIDE_ONLY, "svc/bot.py": PLAIN}) + ai.outcome = _outcome("pairs", [("app", "app/main.py"), ("svc", "svc/bot.py")]) + # The walk's one side row is checked first; the AI rows' check then breaks. + checker.fail_at = 2 + checker.error = FolderCheckerBroken("the child ended") + + assert _scan(_job("archive"), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert len(document["agents"]) == 1 + assert _entries(document) == ["app/main.py"] + assert document["agents"][0]["subfolder"] == "" + assert document["truncated"] is True + assert document["commit_sha"] is None + + +def test_an_upload_with_only_side_rows_keeps_the_walks_row(tmp_path, ai) -> None: + source = _write(tmp_path / "source", {**SIDE_ONLY, "tests/test_other.py": PLAIN}) + ai.outcome = _outcome("pairs", [("tests", "tests/test_other.py")]) + + assert _scan(_job("archive"), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert len(ai.calls) == 1 + assert _entries(document) == ["tests/test_bot.py"] + assert document["agents"][0]["subfolder"] == "" + assert document["truncated"] is True + + +def test_hints_count_an_uploads_other_side_files(tmp_path) -> None: + walk = _walk( + agents=(_candidate("tests/test_bot.py", ""),), + upload_others=2, + upload_others_side=2, + unread_files=0, + ) + + assert "agent-like files inside them: 3;" in _hints(walk) + + +def _beside_a_script_agent(helper: str) -> dict[str, str]: + return {"web/src/agent.ts": SCRIPT_AGENT, "backend/judge.py": HELPERS[helper]} + + +@pytest.mark.parametrize("helper", sorted(HELPERS)) +def test_a_script_agent_beside_a_python_model_helper_is_asked_for(tmp_path, ai, helper) -> None: + source = _write(tmp_path / "source", _beside_a_script_agent(helper)) + ai.outcome = _outcome("pairs", [("web", "web/src/agent.ts")]) + output = tmp_path / "out" + + assert _read_mode("github", source, output) == 0 + + document = _agents(output) + assert len(ai.calls) == 1 + assert document["status"] == "found" + assert _entries(document) == ["backend/judge.py", "web/src/agent.ts"] + assert _read(output, "cost.json")["complete"] is True + + +# Every suffix spelled out, so a suffix dropped from the walk's list fails its own case. +@pytest.mark.parametrize("suffix", [".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs"]) +def test_an_agent_in_any_script_suffix_beside_a_helper_is_asked_for(tmp_path, ai, suffix) -> None: + entry = f"web/agent{suffix}" + source = _write(tmp_path / "source", {entry: SCRIPT_AGENT, "backend/judge.py": MODEL_HELPER}) + ai.outcome = _outcome("pairs", [("web", entry)]) + output = tmp_path / "out" + + assert _read_mode("github", source, output) == 0 + + assert len(ai.calls) == 1 + assert _entries(_agents(output)) == ["backend/judge.py", entry] + + +def test_built_vendored_and_config_scripts_do_not_ask_for_the_model(tmp_path, ai) -> None: + scripts = { + "tailwind.config.js": False, + "eslint.config.mjs": False, + "vite.config.ts": False, + "static/js/jquery.min.js": False, + "web/app.min.js": False, + "frontend/dist/assets/index-3f9a.js": False, + "web/build/main.js": False, + "web/vendor/lib.js": False, + "types/env.d.ts": False, + "types/env.d.mts": False, + "web/src/agent.ts": True, + } + for script, asks in scripts.items(): + shape = script.replace("/", "_") + source = _write(tmp_path / shape, {"app/main.py": CLIENT, script: "export {};\n"}) + output = tmp_path / f"out-{shape}" + ai.calls.clear() + + assert _read_mode("github", source, output) == 0, script + + assert len(ai.calls) == (1 if asks else 0), script + assert _entries(_agents(output)) == ["app/main.py"], script + + +def test_the_model_is_not_asked_past_a_helper_without_a_script_agent_beside_it( + tmp_path, ai +) -> None: + shapes = { + "no_script": ({"backend/judge.py": MODEL_HELPER}, "backend/judge.py"), + "beside_a_framework_agent": ( + {"web/src/agent.ts": SCRIPT_AGENT, "bot/graph.py": GRAPH}, + "bot/graph.py", + ), + "script_only_in_a_side_folder": ( + {"examples/web/agent.ts": SCRIPT_AGENT, "backend/judge.py": MODEL_HELPER}, + "backend/judge.py", + ), + } + for shape, (files, entry) in shapes.items(): + source = _write(tmp_path / shape, files) + output = tmp_path / f"out-{shape}" + + assert _read_mode("github", source, output) == 0, shape + + document = _agents(output) + assert ai.calls == [], shape + assert _entries(document) == [entry], shape + assert _read(output, "cost.json")["complete"] is True, shape + + +@pytest.mark.parametrize("helper", sorted(HELPERS)) +def test_an_upload_prefers_the_models_row_to_a_python_model_helper(tmp_path, ai, helper) -> None: + source = _write(tmp_path / "source", _beside_a_script_agent(helper)) + ai.outcome = _outcome("pairs", [("web", "web/src/agent.ts")]) + output = tmp_path / "out" + + assert _read_mode("archive", source, output) == 0 + + document = _agents(output) + assert len(ai.calls) == 1 + assert _entries(document) == ["web/src/agent.ts"] + assert document["agents"][0]["subfolder"] == "" + assert document["truncated"] is True + + +def test_an_uploads_model_row_in_a_side_folder_loses_to_a_python_client(tmp_path, ai) -> None: + source = _write( + tmp_path / "source", + {**_beside_a_script_agent("client"), "examples/agent.ts": SCRIPT_AGENT}, + ) + ai.outcome = _outcome("pairs", [("examples", "examples/agent.ts")]) + output = tmp_path / "out" + + assert _read_mode("archive", source, output) == 0 + + document = _agents(output) + assert len(ai.calls) == 1 + assert _entries(document) == ["backend/judge.py"] + assert document["truncated"] is True + + +@pytest.mark.parametrize("helper", sorted(HELPERS)) +def test_hints_count_the_model_helpers_outside_side_folders(tmp_path, ai, helper) -> None: + source = _write( + tmp_path / "source", {**_beside_a_script_agent(helper), "tests/test_bot.py": CLIENT} + ) + + assert _read_mode("github", source, tmp_path / "out") == 0 + + hints = ai.calls[0]["hints"] + assert "agent-like files inside them: 1;" in hints + assert "files outside them that only call a model: 1;" in hints + for word in ("judge", "backend", "web", "/", ".py"): + assert word not in hints + + +def test_an_uploads_hints_count_each_unlisted_file_where_it_sits(tmp_path, ai) -> None: + files = { + "a/one.py": CLIENT, + "b/two.py": CLIENT, + "c/three.py": MODEL_HELPER, + "tests/test_four.py": CLIENT, + "web/agent.ts": SCRIPT_AGENT, + } + source = _write(tmp_path / "source", files) + + assert _read_mode("archive", source, tmp_path / "out") == 0 + + hints = ai.calls[0]["hints"] + assert "agent-like files inside them: 1;" in hints + assert "files outside them that only call a model: 3;" in hints + + +def test_the_models_rows_are_kept_beside_fifty_helpers(tmp_path, checker, ai) -> None: + fifty = {f"p{index:02d}/main.py": CLIENT for index in range(50)} + forty_seven = {f"p{index:02d}/main.py": CLIENT for index in range(47)} + scripts = {f"web{index}/agent.ts": SCRIPT_AGENT for index in range(5)} + # Each helper in a folder of its own, so every walk row is `medium` and every model row + # `low`: by confidence alone, the cap would drop the model's rows. + shapes = { + "fifty_and_one": ( + {**fifty, "web0/agent.ts": SCRIPT_AGENT}, + [("web0", "web0/agent.ts")], + ["web0/agent.ts"], + ), + "forty_seven_and_five": ( + {**forty_seven, **scripts}, + [(f"web{index}", f"web{index}/agent.ts") for index in range(5)], + sorted(scripts), + ), + # A model row inside a side folder keeps its place behind every walk row. + "fifty_and_one_in_a_side_folder": ( + {**fifty, "web0/agent.ts": SCRIPT_AGENT, "examples/agent.ts": SCRIPT_AGENT}, + [("examples", "examples/agent.ts")], + [], + ), + } + for shape, (files, pairs, kept) in shapes.items(): + source = _write(tmp_path / shape, files) + output = tmp_path / f"out-{shape}" + ai.calls.clear() + ai.outcome = _outcome("pairs", pairs) + + assert _read_mode("github", source, output) == 0, shape + + document = _agents(output) + written = _entries(document) + assert len(ai.calls) == 1, shape + assert len(written) == 50, shape + assert [entry for entry in written if not entry.startswith("p")] == kept, shape + assert document["truncated"] is True, shape + + +def test_the_models_row_beside_fifty_helpers_survives_a_crash_after_the_model( + tmp_path, checker, ai +) -> None: + files = {f"p{index:02d}/main.py": CLIENT for index in range(50)} + source = _write(tmp_path / "source", {**files, "web/agent.ts": SCRIPT_AGENT}) + ai.outcome = _outcome("pairs", [("web", "web/agent.ts")]) + # The walk's fifty checks pass; the model row's check crashes. + checker.fail_at, checker.error = 51, RuntimeError("the checker broke") + output = tmp_path / "out" + + assert _read_mode("github", source, output) == 0 + + document = _agents(output) + assert len(ai.calls) == 1 + assert "web/agent.ts" in _entries(document) + assert len(document["agents"]) == 50 + assert document["truncated"] is True + + +def test_a_langchain_chat_model_needs_no_ai_step(tmp_path, ai) -> None: + example = Path(__file__).parents[2] / "examples" / "frameworks" / "langchain_agent.py" + source = tmp_path / "source" + source.mkdir() + shutil.copyfile(example, source / "langchain_agent.py") + + assert _scan(_job(), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert ai.calls == [] + assert document["status"] == "found" + assert _entries(document) == ["langchain_agent.py"] + assert document["agents"][0]["confidence"] == "low" + + +def test_every_hazard_look_gets_the_scans_one_memo_and_its_deadline( + tmp_path, monkeypatch, ai +) -> None: + source = _write(tmp_path / "source", {**SIDE_ONLY, "web/main.py": PLAIN}) + ai.outcome = _outcome("pairs", [("app", "app/main.py"), ("web", "web/main.py")]) + looks: list[tuple[str, float, object]] = [] + real = scan_walk.build_hazard + + # The real defaults, so a call that leaves a keyword out is recorded with the default. + def recording(source, subfolder, *, deadline=math.inf, clock=time.monotonic, hazards=None): + looks.append((subfolder, deadline, hazards)) + return real(source, subfolder, deadline=deadline, clock=clock, hazards=hazards) + + monkeypatch.setattr(scan_walk, "build_hazard", recording) + start = time.monotonic() + + assert _scan(_job(), source, tmp_path / "out") == 0 + + end = time.monotonic() + assert len(ai.calls) == 1 + # The walk's side row first, then each row the model found. + assert [subfolder for subfolder, _, _ in looks] == ["tests", "app", "web"] + memo = looks[0][2] + assert isinstance(memo, dict) + assert all(hazards is memo for _, _, hazards in looks) + deadlines = {deadline for _, deadline, _ in looks} + assert len(deadlines) == 1 + [deadline] = deadlines + assert start + agent_scan.CLOCK <= deadline <= end + agent_scan.CLOCK + + +def test_a_file_with_a_chat_model_and_a_client_is_a_low_model_row( + tmp_path, checker, ai +) -> None: + source = _write( + tmp_path / "source", + { + "bot.py": ( + "from langchain_openai import ChatOpenAI\n" + "from openai import OpenAI\n" + "\n" + "ChatOpenAI()\n" + "OpenAI()\n" + ) + }, + ) + checker.name = "langchain" + + assert _scan(_job(), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert ai.calls == [] + [agent] = document["agents"] + assert agent["confidence"] == "low" + assert "Calls a chat model through LangChain in bot.py, line 4." in agent["evidence"] + assert "Creates a client for openai in bot.py, line 5." in agent["evidence"] + + +def test_an_agent_at_a_path_the_platform_cannot_carry_is_never_reported_as_none( + tmp_path, ai +) -> None: + """The vercel/ai-chatbot run: the model found ``app/(chat)/api/chat/route.ts`` and every + submit was refused, which used to be written as a whole list with no agents.""" + source = _write(tmp_path / "source", {"app/main.py": PLAIN}) + + ai.outcome = _outcome("empty", uncarried=1) + assert _scan(_job(), source, tmp_path / "out-empty") == 0 + document = _agents(tmp_path / "out-empty") + assert document["agents"] == [] + assert document["truncated"] is True + assert document["error"]["code"] == "agent_path_unsupported" + + ai.outcome = _outcome("pairs", [("app", "app/main.py")], uncarried=1) + assert _scan(_job(), source, tmp_path / "out-pairs") == 0 + document = _agents(tmp_path / "out-pairs") + assert _entries(document) == ["app/main.py"] + assert document["truncated"] is True + assert document["error"] is None + + +@pytest.mark.parametrize("step", ["run", "result"]) +def test_a_fault_in_the_walk_is_a_crash_not_a_big_repository(tmp_path, monkeypatch, ai, step) -> None: + """A bug in the walk must not be told to the person as "the repository is too large".""" + source = _write(tmp_path / "source", {"app/graph.py": GRAPH}) + + def broken(*args, **kwargs): + raise KeyError("a fault in the kit") + + monkeypatch.setattr(scan_walk._Walk, step, broken) + output = tmp_path / "out" + + assert _read_mode("github", source, output) == 1 + + failure = _read(output, "failure.json") + assert (failure["code"], failure["retryable"]) == ("scan_crashed", True) + assert not (output / "agents.json").exists() + assert len(ai.calls) == 0 + + +def test_the_scan_reads_its_clock_variable_when_it_starts(tmp_path, monkeypatch, ai) -> None: + """The helper's hard stop and the scan's own clock read ALK_SCAN_MAX_SECONDS at the same + moment, so a shorter clock set at run time ends the scan in its own file.""" + source = _write(tmp_path / "source", {"a/graph.py": GRAPH}) + monkeypatch.setattr(agent_scan, "CLOCK", 1200.0) + monkeypatch.setenv("ALK_SCAN_MAX_SECONDS", "0.000001") + + assert _scan(_job(), source, tmp_path / "out") == 0 + + assert _agents(tmp_path / "out")["error"]["code"] == "scan_timed_out" + assert ai.calls == [] + + +def test_every_scan_logger_masks_secrets() -> None: + from fi.alk.harness import scan_ai, scan_folders + from fi.alk.harness.reading_entry import MaskSecrets + + for module in (agent_scan, scan_walk, scan_folders, scan_ai): + assert any(isinstance(f, MaskSecrets) for f in module.logger.filters), module.__name__ + + +def test_the_model_steps_limit_is_read_when_it_is_asked(tmp_path, monkeypatch, ai) -> None: + source = _write(tmp_path / "source", SIDE_ONLY) + monkeypatch.setenv(agent_scan.AI_ENV, "100") + + assert _scan(_job(), source, tmp_path / "out") == 0 + + assert [call["seconds"] for call in ai.calls] == [100.0] diff --git a/tests/harness/test_authoring_entrypoint_find_agents.py b/tests/harness/test_authoring_entrypoint_find_agents.py new file mode 100644 index 000000000..91236af53 --- /dev/null +++ b/tests/harness/test_authoring_entrypoint_find_agents.py @@ -0,0 +1,241 @@ +"""The `--find-agents` switch: it reaches the scan, it can't be given with the reading's switch, +and a repository scanned through it leaves a file the platform's reader accepts.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from fi.alk.harness import agent_scan, reading_entry, spend +from fi.alk.harness.agent_scan import FIND_AGENTS +from fi.alk.harness.authoring_entrypoint import main +from fi.alk.harness.scan_ai import AiOutcome +from fi.alk.harness.scan_document import wire_problems +from fi.alk.harness.scan_folders import FolderWord + +GRAPH = "from langgraph.graph import StateGraph\nStateGraph(dict)\n" +CLIENT = "from openai import OpenAI\nOpenAI()\n" +COMMIT = "CD" * 20 + + +class FakeChecker: + def __enter__(self) -> FakeChecker: + return self + + def __exit__(self, *exc: object) -> None: + return None + + def check(self, source, subfolder, *, deadline, clock=None) -> FolderWord: + return FolderWord({"name": "langgraph", "version": None, "label": None}, finished=True) + + +@pytest.fixture(autouse=True) +def _fresh_state(monkeypatch, tmp_path): + monkeypatch.delenv(spend.JOURNAL_ALIAS, raising=False) + monkeypatch.setattr(spend, "_path", None) + # The machine's own secrets files must not decide what a test sees masked. + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (tmp_path / "no-secrets.json",)) + spend._stages.clear() + yield + spend._stages.clear() + + +@pytest.fixture +def fakes(monkeypatch) -> list[dict]: + """The folder checker and the model step, faked; the model's calls are returned.""" + asked: list[dict] = [] + + async def no_model(source, *, hints, seconds, on_turn_end) -> AiOutcome: + asked.append({"hints": hints}) + return AiOutcome("empty", (), 0, True, False, "fixed words") + + monkeypatch.setattr(agent_scan, "FolderChecker", FakeChecker) + monkeypatch.setattr(agent_scan, "run_ai_step", no_model) + return asked + + +def _write(folder: Path, files: dict[str, str]) -> Path: + for relative, content in files.items(): + path = folder / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return folder + + +_SOURCES = { + "github": { + "kind": "github", + "visibility": "public", + "repository": "acme/support-bot", + "commit_sha": COMMIT, + }, + "archive": {"kind": "archive", "archive_artifact_id": "upload-1"}, +} + + +def _job_file(tmp_path: Path, kind: str = "github") -> Path: + path = tmp_path / f"job-{kind}.json" + path.write_text( + json.dumps( + { + "job_id": "job", + "run_id": "run", + "execution": "hosted", + "source": _SOURCES[kind], + "runtime": {"isolation": "dedicated_vm"}, + "agent": {"connector": "auto"}, + } + ), + encoding="utf-8", + ) + return path + + +def _found(output: Path) -> dict: + document = json.loads((output / "agents.json").read_text(encoding="utf-8")) + assert wire_problems(document) == [] + return document + + +def test_the_switch_reaches_the_scan(tmp_path, monkeypatch) -> None: + calls = [] + + def recording(mode, *, job_path, source, output): + calls.append((mode, job_path, source, output)) + return 0 + + monkeypatch.setattr(reading_entry, "run_reading_mode", recording) + job = _job_file(tmp_path) + + code = main( + [ + str(job), + "--source", + str(tmp_path / "a" / ".." / "source"), + "--output", + str(tmp_path / "a" / ".." / "reading"), + "--find-agents", + ] + ) + + assert code == 0 + assert calls == [ + ( + FIND_AGENTS, + job, + (tmp_path / "source").resolve(), + (tmp_path / "reading").resolve(), + ) + ] + + +def test_the_two_switches_exclude_each_other(tmp_path, monkeypatch) -> None: + calls = [] + monkeypatch.setattr( + reading_entry, "run_reading_mode", lambda mode, **kwargs: calls.append(mode) or 0 + ) + + with pytest.raises(SystemExit) as stopped: + main( + [ + str(_job_file(tmp_path)), + "--source", + str(tmp_path / "source"), + "--output", + str(tmp_path / "reading"), + "--read-agent-only", + "--find-agents", + ] + ) + + assert stopped.value.code == 2 + assert calls == [] + + +def test_a_github_repo_gives_found(tmp_path, fakes) -> None: + source = _write( + tmp_path / "source", + {"graph.py": GRAPH, "pyproject.toml": '[project]\nname = "bot"\n'}, + ) + output = tmp_path / "reading" + + code = main( + [ + str(_job_file(tmp_path)), + "--source", + str(source), + "--output", + str(output), + "--find-agents", + ] + ) + + assert code == 0 + document = _found(output) + assert document["status"] == "found" + assert document["commit_sha"] == COMMIT.lower() + assert [(agent["entry"], agent["subfolder"]) for agent in document["agents"]] == [ + ("graph.py", "") + ] + assert document["agents"][0]["name"] == "support-bot" + assert document["truncated"] is False + assert json.loads((output / "cost.json").read_text(encoding="utf-8"))["complete"] is True + assert not (output / "failure.json").exists() + assert fakes == [] + + +def test_fifty_one_agents_give_fifty_and_truncated(tmp_path, fakes) -> None: + source = _write(tmp_path / "source", {f"bot_{index:02d}.py": CLIENT for index in range(51)}) + output = tmp_path / "reading" + + code = main( + [ + str(_job_file(tmp_path)), + "--source", + str(source), + "--output", + str(output), + "--find-agents", + ] + ) + + assert code == 0 + document = _found(output) + assert document["status"] == "found" + assert len(document["agents"]) == 50 + assert document["truncated"] is True + assert fakes == [] + + +def test_an_upload_with_nothing_agent_like_is_none(tmp_path, fakes) -> None: + # Nothing that looks like code, so no model is asked; then plain code, so the model is + # asked and finds nothing. + shapes = { + "quiet": ({"README.md": "# notes\n"}, 0), + "plain": ({"app/main.py": "def main():\n return 1\n"}, 1), + } + for shape, (files, asked) in shapes.items(): + fakes.clear() + source = _write(tmp_path / shape, files) + output = tmp_path / f"reading-{shape}" + + code = main( + [ + str(_job_file(tmp_path, "archive")), + "--source", + str(source), + "--output", + str(output), + "--find-agents", + ] + ) + + assert code == 0, shape + document = _found(output) + assert (document["status"], document["agents"], document["error"]) == ("none", [], None) + assert document["commit_sha"] is None + assert len(fakes) == asked, shape + assert (output / "cost.json").exists() + assert not (output / "failure.json").exists() diff --git a/tests/harness/test_reading_entry.py b/tests/harness/test_reading_entry.py new file mode 100644 index 000000000..a5a062076 --- /dev/null +++ b/tests/harness/test_reading_entry.py @@ -0,0 +1,798 @@ +"""The shared way into a reading: what it refuses, the cost it leaves, and its time limit.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from fi.alk.harness import reading_entry, spend +from fi.alk.harness.job import FailureDomain, HarnessFailure, HarnessJob, HarnessStage +from fi.alk.harness.reading_entry import ( + MESSAGE_LIMIT, + READING_JOB_INVALID, + READING_OUTPUT_NOT_EMPTY, + ReadingMode, + run_reading_mode, + write_reading_cost, + write_reading_failure, +) + +_DEFAULT_SECRET_FILES = reading_entry._SECRET_FILES + +READ_AGENT_VALUES = { + "name": "read_agent", + "time_limit_seconds": 2100.0, + "timed_out_code": "agent_reading_timed_out", + "timed_out_retryable": False, + "crashed_code": "agent_reading_crashed", +} +FIND_AGENTS_VALUES = { + "name": "find_agents", + "time_limit_seconds": 1800.0, + "timed_out_code": "scan_hard_limit_reached", + "timed_out_retryable": False, + "crashed_code": "scan_crashed", +} + + +class _LoudExit: + """``os`` for the helper under test: its hard limit ending this test process is a failure, + never the silent exit 0 its backstop gives a finished reading.""" + + def __getattr__(self, name): + return getattr(os, name) + + @staticmethod + def _exit(_code): + os.write(2, b"a reading's hard limit ended the test process\n") + os._exit(97) + + +@pytest.fixture(autouse=True) +def _fresh_state(monkeypatch, tmp_path): + monkeypatch.setattr(reading_entry, "os", _LoudExit()) + monkeypatch.delenv(spend.JOURNAL_ALIAS, raising=False) + # The machine's own secrets files must not decide what a test sees masked. + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (tmp_path / "no-secrets.json",)) + spend._stages.clear() + spend.journal_to(None) + yield + spend._stages.clear() + spend.journal_to(None) + + +@pytest.fixture(params=[READ_AGENT_VALUES, FIND_AGENTS_VALUES], ids=["read_agent", "find_agents"]) +def values(request): + return request.param + + +def _job(source: dict | None = None) -> dict: + return { + "job_id": "job", + "run_id": "run", + "execution": "hosted", + "source": source + or { + "kind": "github", + "visibility": "public", + "repository": "acme/agent", + "commit_sha": "a" * 40, + }, + "runtime": {"isolation": "dedicated_vm"}, + "agent": {"connector": "auto"}, + } + + +class _Work: + """A mode's work that remembers being called; ``action`` stands in for what it does.""" + + def __init__(self, code: int = 0, action=None) -> None: + self.code = code + self.action = action + self.calls: list[tuple] = [] + + async def __call__(self, job, source, output) -> int: + self.calls.append((job, source, output)) + if self.action is not None: + self.action(job, source, output) + return self.code + + +def _accept(_parsed) -> None: + return None + + +def _mode(values: dict, work, check_job=_accept, **overrides) -> ReadingMode: + return ReadingMode(check_job=check_job, work=work, **{**values, **overrides}) + + +def _run(mode: ReadingMode, root: Path, job_path: Path | None = None) -> int: + if job_path is None: + job_path = root / "job.json" + job_path.write_text(json.dumps(_job()), encoding="utf-8") + return run_reading_mode( + mode, job_path=job_path, source=root / "source", output=root / "reading" + ) + + +def _failure_in(output: Path) -> dict: + return json.loads((output / "failure.json").read_text(encoding="utf-8")) + + +def _cost_in(output: Path) -> dict: + return json.loads((output / "cost.json").read_text(encoding="utf-8")) + + +def _shape(failure: dict) -> dict: + return {key: failure[key] for key in ("domain", "stage", "code", "retryable")} + + +def _refusal(code: str = "scan_job_invalid") -> HarnessFailure: + return HarnessFailure( + domain=FailureDomain.PLATFORM_SYNC, + stage=HarnessStage.UNDERSTANDING_AGENT, + code=code, + message="A scan reads a whole repository, not one folder of it.", + retryable=False, + details={"subfolder": "agents/support"}, + ) + + +# The helper ends the process with os._exit at its limit, so those tests run it in a child. +_CHILD = r""" +import asyncio, json, logging, shutil, sys, time +config = json.loads(sys.argv[1]) +sys.path[:0] = config["sys_path"] +logging.basicConfig(level=logging.INFO) +from pathlib import Path +from fi.alk.harness import reading_entry, spend +from fi.alk.harness.reading_entry import ReadingMode, run_reading_mode + +reading_entry._SECRET_FILES = () +root = Path(config["root"]) + + +def broken(*_args, **_kwargs): + raise RuntimeError("this writer is broken") + + +async def work(job, source, output): + if config["work"] == "returns": + return 0 + if config["work"] == "leaves_a_thread": + (output / "agent.json").write_text('{"agent": "kept"}') + reading_entry.write_reading_cost(output, complete=True) + try: + # A scan given up on by its caller: the thread itself runs on past the limit. + await asyncio.wait_for(asyncio.to_thread(time.sleep, 30.0), 0.1) + except TimeoutError: + pass + return 0 + if config["work"] in ("completes_then_blocks", "completes_then_raises"): + spend.record("read-agent", 0.3, turns=1, models={"m"}) + reading_entry.write_reading_cost(output, complete=True) + if config["work"] == "completes_then_raises": + raise RuntimeError("the work broke after its last write") + if config["work"] == "spoils_the_output_then_blocks": + # Every write at the limit now fails, and each failure would log. + shutil.rmtree(output) + output.write_text("a file, not a folder") + if config["work"] == "records_then_blocks": + spend.record("read-agent", 0.21, turns=1, models={"m"}) + if config["work"] == "breaks_the_writers_then_blocks": + reading_entry.write_reading_failure = broken + reading_entry.write_reading_cost = broken + await asyncio.to_thread(time.sleep, config["block_seconds"]) + return 0 + + +mode = ReadingMode(check_job=lambda job: None, work=work, **config["mode"]) +(root / "started").write_text(repr(time.time())) +code = run_reading_mode( + mode, job_path=root / "job.json", source=root / "source", output=root / "reading" +) +time.sleep(config["linger_seconds"]) +sys.exit(code) +""" + +_BLOCK_SECONDS = 20.0 + + +def _run_in_a_child(root: Path, mode_values: dict, work: str, *, linger: float = 0.0): + """Returns the finished child and the seconds from the helper's start to the child's end.""" + root.mkdir(parents=True, exist_ok=True) + (root / "job.json").write_text(json.dumps(_job()), encoding="utf-8") + config = { + "sys_path": [entry for entry in sys.path if entry], + "root": str(root), + "mode": mode_values, + "work": work, + "block_seconds": _BLOCK_SECONDS, + "linger_seconds": linger, + } + env = { + **os.environ, + "OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:9", + "OTEL_TRACES_EXPORTER": "none", + } + env.pop(spend.JOURNAL_ALIAS, None) + finished = subprocess.run( + [sys.executable, "-c", _CHILD, json.dumps(config)], + env=env, + capture_output=True, + text=True, + timeout=_BLOCK_SECONDS + 40, + ) + ended = time.time() + started = float((root / "started").read_text(encoding="utf-8")) + return finished, ended - started + + +@pytest.mark.parametrize("case", ["missing", "not_json", "not_a_job", "folder_on_an_upload"]) +def test_an_unreadable_job_is_refused_before_any_work(tmp_path, values, case): + job_path = tmp_path / "job.json" + if case == "not_json": + job_path.write_text("{not json", encoding="utf-8") + elif case == "not_a_job": + job_path.write_text(json.dumps({"hello": "world"}), encoding="utf-8") + elif case == "folder_on_an_upload": + upload = {"kind": "archive", "archive_artifact_id": "upload-1"} + # Without the folder the same upload is a good job, so only the folder is refused. + HarnessJob.model_validate(_job(upload)) + job_path.write_text( + json.dumps(_job({**upload, "subfolder": "agents/support"})), encoding="utf-8" + ) + checked: list = [] + work = _Work() + + code = _run(_mode(values, work, check_job=checked.append), tmp_path, job_path) + + output = tmp_path / "reading" + assert code == 78 + assert _shape(_failure_in(output)) == { + "domain": "platform_sync", + "stage": "understanding_agent", + "code": READING_JOB_INVALID, + "retryable": False, + } + assert not (output / "cost.json").exists() + assert checked == [] + assert work.calls == [] + + +_TOKEN = "ghp_REVIEWSECRET1234567890abcdef" + + +@pytest.mark.parametrize( + "refused", + [ + {**_job(), "metadata": {"token": _TOKEN}}, + # pydantic's own message for this one repeats the value whole. + {**_job(), "schema_version": _TOKEN}, + # A dict's key is in the error's location. + {**_job(), "agent": {"connector": "auto", "secret_refs": {_TOKEN: "not-an-object"}}}, + ], + ids=["secret_field", "schema_version", "secret_refs_key"], +) +def test_a_refused_job_leaves_none_of_its_values_anywhere( + tmp_path, values, capfd, caplog, refused +): + token = _TOKEN + job_path = tmp_path / "job.json" + job_path.write_text(json.dumps(refused), encoding="utf-8") + + with caplog.at_level("DEBUG"): + assert _run(_mode(values, _Work()), tmp_path, job_path) == 78 + + failure = (tmp_path / "reading" / "failure.json").read_text(encoding="utf-8") + assert json.loads(failure)["code"] == READING_JOB_INVALID + captured = capfd.readouterr() + kept = failure + captured.out + captured.err + caplog.text + assert not [token[at : at + 8] for at in range(len(token) - 7) if token[at : at + 8] in kept] + + +def test_a_job_nested_past_the_parsers_depth_is_refused(tmp_path, values): + job_path = tmp_path / "job.json" + job_path.write_text("[" * 100_000 + "]" * 100_000, encoding="utf-8") + work = _Work() + + assert _run(_mode(values, work), tmp_path, job_path) == 78 + + failure = _failure_in(tmp_path / "reading") + assert (failure["code"], failure["message"]) == ( + READING_JOB_INVALID, + "The job file could not be read as a harness job (RecursionError).", + ) + assert work.calls == [] + + +def test_the_modes_own_job_check_refuses_before_any_work(tmp_path, values): + seen: list = [] + + def check_job(job): + seen.append(job) + return _refusal() + + work = _Work() + + assert _run(_mode(values, work, check_job=check_job), tmp_path) == 78 + + output = tmp_path / "reading" + assert _failure_in(output) == _refusal().model_dump(mode="json") + assert [type(job) for job in seen] == [HarnessJob] + assert seen[0].job_id == "job" + assert not (output / "cost.json").exists() + assert work.calls == [] + + +def test_a_job_check_that_raises_is_a_crash_before_any_work(tmp_path, values): + def check_job(_job): + raise KeyError("metadata") + + work = _Work() + + assert _run(_mode(values, work, check_job=check_job), tmp_path) == 1 + + output = tmp_path / "reading" + failure = _failure_in(output) + assert _shape(failure) == { + "domain": "infrastructure", + "stage": "understanding_agent", + "code": values["crashed_code"], + "retryable": True, + } + assert "KeyError" in failure["message"] + assert not (output / "cost.json").exists() + assert work.calls == [] + + +def test_a_non_empty_output_is_refused_and_left_alone(tmp_path, values): + old = tmp_path / "old" + (old / "reading").mkdir(parents=True) + (old / "reading" / "contract.json").write_bytes(b'{"old": true}') + stale = tmp_path / "stale" + (stale / "reading").mkdir(parents=True) + stale_failure = b'{"code": "from_an_earlier_run"}' + (stale / "reading" / "failure.json").write_bytes(stale_failure) + a_file = tmp_path / "a_file" + a_file.mkdir() + (a_file / "reading").write_bytes(b"not a folder") + work = _Work() + + assert _run(_mode(values, work), old) == 78 + assert _run(_mode(values, work), stale) == 78 + assert _run(_mode(values, work), a_file) == 78 + + assert (old / "reading" / "contract.json").read_bytes() == b'{"old": true}' + assert sorted(entry.name for entry in (old / "reading").iterdir()) == [ + "contract.json", + "failure.json", + ] + assert _shape(_failure_in(old / "reading")) == { + "domain": "platform_sync", + "stage": "understanding_agent", + "code": READING_OUTPUT_NOT_EMPTY, + "retryable": False, + } + assert [entry.name for entry in (stale / "reading").iterdir()] == ["failure.json"] + assert (stale / "reading" / "failure.json").read_bytes() == stale_failure + assert (a_file / "reading").read_bytes() == b"not a folder" + assert work.calls == [] + + +def test_cost_starts_at_zero_and_incomplete_before_the_work(tmp_path, values): + seen: dict = {} + + def look(_job, _source, output): + seen.update(_cost_in(output)) + + assert _run(_mode(values, _Work(action=look)), tmp_path) == 0 + + assert seen["total_usd"] == 0 + assert seen["unpriced_turns"] == 0 + assert seen["stages"] == [] + assert seen["complete"] is False + + +def test_a_failed_first_cost_write_stops_before_the_work(tmp_path, values, monkeypatch): + blocked = tmp_path / "blocked" + blocked.write_text("a file, not a folder", encoding="utf-8") + assert write_reading_cost(blocked, complete=False) is False + + monkeypatch.setattr(reading_entry, "write_reading_cost", lambda output, *, complete: False) + work = _Work() + + assert _run(_mode(values, work), tmp_path) == 1 + + assert work.calls == [] + assert _shape(_failure_in(tmp_path / "reading")) == { + "domain": "infrastructure", + "stage": "understanding_agent", + "code": values["crashed_code"], + "retryable": True, + } + + +@pytest.mark.parametrize("retryable, exit_code", [(False, 78), (True, 1)]) +def test_the_time_limit_writes_its_code_and_ends_the_process( + tmp_path, values, retryable, exit_code +): + limit = 1.0 + mode_values = {**values, "time_limit_seconds": limit, "timed_out_retryable": retryable} + + finished, elapsed = _run_in_a_child(tmp_path, mode_values, "blocks") + + output = tmp_path / "reading" + assert finished.returncode == exit_code, finished.stderr + assert limit <= elapsed <= limit + 5 + assert _shape(_failure_in(output)) == { + "domain": "infrastructure", + "stage": "understanding_agent", + "code": values["timed_out_code"], + "retryable": retryable, + } + assert _cost_in(output)["complete"] is False + assert "reading started at " in finished.stderr + + +def test_the_time_limit_ends_the_process_even_when_its_writes_fail(tmp_path): + limit = 1.0 + mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": limit} + + finished, elapsed = _run_in_a_child(tmp_path, mode_values, "breaks_the_writers_then_blocks") + + assert finished.returncode == 78, finished.stderr + assert limit <= elapsed <= limit + 5 + # Proof the writers really raised: the limit could leave no failure of its own. + assert not (tmp_path / "reading" / "failure.json").exists() + + +def test_the_time_limit_does_not_fire_after_the_work_returns(tmp_path): + mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} + + finished, elapsed = _run_in_a_child(tmp_path, mode_values, "returns", linger=2.0) + + assert finished.returncode == 0, finished.stderr + assert elapsed >= 2.0 + assert not (tmp_path / "reading" / "failure.json").exists() + + +def test_a_limit_named_by_the_environment_is_read_when_the_reading_starts( + tmp_path, monkeypatch +): + monkeypatch.setenv("ALK_TEST_READING_LIMIT", "0.5") + mode_values = { + **READ_AGENT_VALUES, + "time_limit_seconds": 30.0, + "limit_env": "ALK_TEST_READING_LIMIT", + "limit_margin_seconds": 0.5, + } + + finished, elapsed = _run_in_a_child(tmp_path, mode_values, "blocks") + + assert finished.returncode == 78, finished.stderr + assert 1.0 <= elapsed <= 1.0 + 5 + failure = _failure_in(tmp_path / "reading") + assert failure["code"] == "agent_reading_timed_out" + assert "past its 1 s limit" in failure["message"] + + +def test_a_thread_left_after_the_work_ends_the_process_quietly_at_the_limit(tmp_path): + mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} + + finished, elapsed = _run_in_a_child(tmp_path, mode_values, "leaves_a_thread") + + output = tmp_path / "reading" + # The work's own code, at the limit, not after the 30 s thread; and nothing written over the + # files the work left. + assert finished.returncode == 0, finished.stderr + assert 1.0 <= elapsed <= 1.0 + 5 + assert not (output / "failure.json").exists() + assert (output / "agent.json").read_text() == '{"agent": "kept"}' + assert _cost_in(output) == {**_empty_spend(), "complete": True} + + +def _empty_spend() -> dict: + return {"schema": "futureagi.harness-spend.v1", "total_usd": 0.0, "unpriced_turns": 0, "stages": []} + + +def test_a_timeout_after_a_complete_cost_marks_it_incomplete(tmp_path): + mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} + + finished, _ = _run_in_a_child(tmp_path, mode_values, "completes_then_blocks") + + assert finished.returncode == 78, finished.stderr + cost = _cost_in(tmp_path / "reading") + assert (cost["total_usd"], cost["complete"]) == (0.3, False) + assert _failure_in(tmp_path / "reading")["code"] == "agent_reading_timed_out" + + +def test_a_crash_after_a_complete_cost_marks_it_incomplete(tmp_path): + finished, _ = _run_in_a_child(tmp_path, READ_AGENT_VALUES, "completes_then_raises") + + assert finished.returncode == 1, finished.stderr + cost = _cost_in(tmp_path / "reading") + assert (cost["total_usd"], cost["complete"]) == (0.3, False) + assert _failure_in(tmp_path / "reading")["code"] == "agent_reading_crashed" + + +def test_the_way_out_at_the_limit_logs_nothing(tmp_path): + mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} + + finished, elapsed = _run_in_a_child(tmp_path, mode_values, "spoils_the_output_then_blocks") + + assert finished.returncode == 78, finished.stderr + assert elapsed <= 1.0 + 5 + # The writes did fail, and a write that fails logs everywhere else. + assert (tmp_path / "reading").is_file() + assert "reading started at " in finished.stderr + assert "not written" not in finished.stderr + + +def test_a_crash_writes_its_code_with_no_secret_and_a_capped_message(tmp_path, values): + def crash(*_args): + raise RuntimeError("could not fetch https://user:pw@host/repo.git " + "x" * 3000) + + assert _run(_mode(values, _Work(action=crash)), tmp_path) == 1 + + output = tmp_path / "reading" + failure = _failure_in(output) + assert _shape(failure) == { + "domain": "infrastructure", + "stage": "understanding_agent", + "code": values["crashed_code"], + "retryable": True, + } + assert failure["message"].startswith( + "RuntimeError: could not fetch https://user:***@host/repo.git " + ) + assert "pw" not in failure["message"] + assert MESSAGE_LIMIT == 2000 + assert len(failure["message"]) == MESSAGE_LIMIT + assert _cost_in(output)["complete"] is False + + blocked = tmp_path / "blocked" + blocked.write_text("a file, not a folder", encoding="utf-8") + assert write_reading_failure(_refusal(), blocked) is None + + +def test_a_secret_is_masked_in_the_shapes_a_trace_shows_it(tmp_path, monkeypatch, caplog): + pem = "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEF\n-----END PRIVATE KEY-----\n" + credential = json.dumps({"type": "service_account", "private_key": pem}) + secrets = tmp_path / "secrets.json" + secrets.write_text(json.dumps({"GOOGLE_CREDENTIALS_JSON": credential}), encoding="utf-8") + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (secrets,)) + + def crash(*_args): + raise RuntimeError(f"bad key {pem!r}; line MIIEvQIBADANBgkqhkiG9w0BAQEF") + + with caplog.at_level("INFO", logger=reading_entry.__name__): + assert _run(_mode(READ_AGENT_VALUES, _Work(action=crash)), tmp_path) == 1 + + kept = _failure_in(tmp_path / "reading")["message"] + caplog.text + assert "MIIEvQIBADANBgkqhkiG9w0BAQEF" not in kept + assert "BEGIN PRIVATE KEY" not in kept + # The whole value is still masked first; a value under 8 characters, or in neither file, is + # one the mask can't know. + assert reading_entry._read_secret_values((secrets,))[0] in (credential, json.dumps(credential)[1:-1]) + + +def test_each_reading_reads_the_secrets_again(tmp_path, monkeypatch): + secrets = tmp_path / "secrets.json" + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (secrets,)) + for run, value in enumerate(("first-secret-value", "second-secret-value")): + secrets.write_text(json.dumps({"KEY": value}), encoding="utf-8") + root = tmp_path / f"run{run}" + root.mkdir() + + def crash(*_args, said=value): + raise RuntimeError(f"refused {said}") + + assert _run(_mode(READ_AGENT_VALUES, _Work(action=crash)), root) == 1 + assert _failure_in(root / "reading")["message"] == "RuntimeError: refused ***" + + +def test_a_value_from_the_secrets_file_is_masked(tmp_path, monkeypatch, values): + target = "sk-0123456789abcdefg" + simulator = "sim-0123456789abcdef" + assert len(target) == len(simulator) == 20 + secrets = tmp_path / "secrets.json" + secrets.write_text(json.dumps({"TARGET_KEY": target}), encoding="utf-8") + simulator_secrets = tmp_path / "simulator-secrets.json" + simulator_secrets.write_text(json.dumps({"SIMULATOR_KEY": simulator}), encoding="utf-8") + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (secrets, simulator_secrets)) + + def crash(*_args): + raise RuntimeError(f"the provider refused {target} and {simulator}") + + assert _run(_mode(values, _Work(action=crash)), tmp_path) == 1 + + assert _failure_in(tmp_path / "reading")["message"] == ( + "RuntimeError: the provider refused *** and ***" + ) + assert _DEFAULT_SECRET_FILES == ( + Path("/run/futureagi/secrets.json"), + Path("/run/futureagi/simulator-secrets.json"), + ) + + +@pytest.mark.parametrize("exit_code", [0, 1, 78]) +def test_the_works_exit_code_is_returned(tmp_path, values, exit_code): + assert _run(_mode(values, _Work(code=exit_code)), tmp_path) == exit_code + assert not (tmp_path / "reading" / "failure.json").exists() + + +def test_a_works_own_incomplete_cost_is_kept(tmp_path, values): + written: list[int] = [] + + def account(_job, _source, output): + spend.record("read-agent", 0.42, turns=1, models={"m"}) + assert write_reading_cost(output, complete=False) + # Every write moves a new file into place, so a rewrite after the work would show. + written.append((output / "cost.json").stat().st_ino) + + assert _run(_mode(values, _Work(action=account)), tmp_path) == 0 + + output = tmp_path / "reading" + assert (output / "cost.json").stat().st_ino == written[0] + cost = _cost_in(output) + assert cost["total_usd"] == 0.42 + assert cost["complete"] is False + + +def test_each_turn_after_the_first_write_lands_in_the_cost_file(tmp_path): + output = tmp_path / "reading" + assert write_reading_cost(output, complete=False) is True + + spend.record("read-agent", 0.25, turns=1, models={"m"}) + + assert _cost_in(output) == {**spend.snapshot(), "complete": False} + assert sorted(entry.name for entry in output.iterdir()) == ["cost.json"] + + +def test_cost_is_left_after_a_timeout_and_a_crash(tmp_path): + def record_then_crash(*_args): + spend.record("read-agent", 0.21, turns=1, models={"m"}) + raise RuntimeError("the reading broke") + + crashed = tmp_path / "crash" + crashed.mkdir() + assert _run(_mode(READ_AGENT_VALUES, _Work(action=record_then_crash)), crashed) == 1 + cost = _cost_in(crashed / "reading") + assert cost["total_usd"] == 0.21 + assert cost["complete"] is False + assert _failure_in(crashed / "reading")["code"] == "agent_reading_crashed" + + timed_out = tmp_path / "timeout" + mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} + finished, _ = _run_in_a_child(timed_out, mode_values, "records_then_blocks") + assert finished.returncode == 78, finished.stderr + cost = _cost_in(timed_out / "reading") + assert cost["total_usd"] == 0.21 + assert cost["complete"] is False + assert _failure_in(timed_out / "reading")["code"] == "agent_reading_timed_out" + + +def test_the_checks_run_in_order(tmp_path, values): + unreadable = tmp_path / "unreadable" + (unreadable / "reading").mkdir(parents=True) + (unreadable / "reading" / "old.txt").write_text("old", encoding="utf-8") + bad_job = unreadable / "job.json" + bad_job.write_text("{not json", encoding="utf-8") + refused = tmp_path / "refused" + (refused / "reading").mkdir(parents=True) + (refused / "reading" / "old.txt").write_text("old", encoding="utf-8") + work = _Work() + + assert _run(_mode(values, work), unreadable, bad_job) == 78 + assert _run(_mode(values, work, check_job=lambda _parsed: _refusal()), refused) == 78 + + assert _failure_in(unreadable / "reading")["code"] == READING_JOB_INVALID + assert _failure_in(refused / "reading")["code"] == "scan_job_invalid" + assert not (unreadable / "reading" / "cost.json").exists() + assert not (refused / "reading" / "cost.json").exists() + assert work.calls == [] + + +def test_a_crash_keeps_its_trace_in_the_log_masked_and_out_of_the_file( + tmp_path, monkeypatch, caplog +): + secret = "sk-0123456789abcdefg" + secrets = tmp_path / "secrets.json" + secrets.write_text(json.dumps({"TARGET_KEY": secret}), encoding="utf-8") + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (secrets,)) + + def crash(*_args): + raise RuntimeError(f"the provider refused {secret}") + + with caplog.at_level("INFO", logger=reading_entry.__name__): + assert _run(_mode(READ_AGENT_VALUES, _Work(action=crash)), tmp_path) == 1 + + assert "Traceback" not in _failure_in(tmp_path / "reading")["message"] + assert "reading started at " in caplog.text + assert "Traceback (most recent call last)" in caplog.text + assert "RuntimeError: the provider refused ***" in caplog.text + assert secret not in caplog.text + + +def test_each_swallowed_error_is_logged_with_its_trace(tmp_path, monkeypatch, caplog): + blocked = tmp_path / "blocked" + blocked.write_text("a file, not a folder", encoding="utf-8") + + with caplog.at_level("WARNING"): + assert write_reading_failure(_refusal(), blocked) is None + assert write_reading_cost(blocked, complete=False) is False + monkeypatch.setitem(sys.modules, "fi.alk.harness.hosted_entrypoint", None) + assert reading_entry._read_secret_values((tmp_path / "unread.json",)) == () + + traced = [ + record.getMessage() + for record in caplog.records + if record.levelname == "WARNING" and (record.exc_info or record.exc_text) + ] + for said in ("failure file not written", "spend file not written", "secret values not read"): + assert any(said in line for line in traced), (said, traced) + + +def test_cost_json_keeps_the_build_keys_and_adds_complete(tmp_path): + spend.record( + "read-agent", + 0.125, + turns=3, + models={"claude-x"}, + tokens_in=1000, + tokens_out=200, + tokens_cached=100, + tools={"submit_contract": 1}, + refusals={"agent_type": 1}, + ) + output = tmp_path / "reading" + + assert write_reading_cost(output, complete=True) is True + + snapshot = spend.snapshot() + assert "complete" not in snapshot + assert _cost_in(output) == {**snapshot, "complete": True} + assert [entry.name for entry in output.iterdir()] == ["cost.json"] + + +def test_an_unpriced_turn_marks_the_cost_incomplete(tmp_path): + spend.record("read-agent", 0.1, turns=1, models={"m"}) + spend.record("read-agent", None, turns=1, models={"m"}) + output = tmp_path / "reading" + + assert write_reading_cost(output, complete=True) is True + + cost = _cost_in(output) + assert cost["unpriced_turns"] == 1 + assert cost["total_usd"] == 0.1 + assert cost["complete"] is False + + +def test_a_failed_cost_write_leaves_no_half_file(tmp_path, monkeypatch): + output = tmp_path / "reading" + output.mkdir() + + def refuses(*_args, **_kwargs): + raise OSError("no space left on the device") + + monkeypatch.setattr(os, "replace", refuses) + + assert write_reading_cost(output, complete=False) is False + assert list(output.iterdir()) == [] + + +def test_a_builds_spend_file_gains_no_complete_key(tmp_path): + assert write_reading_cost(tmp_path / "reading", complete=True) is True + build = tmp_path / "authoring" / "cost.json" + + spend.journal_to(build) + spend.record("understand-agent", 0.1, turns=1, models={"m"}) + + assert json.loads(build.read_text(encoding="utf-8")) == spend.snapshot() diff --git a/tests/harness/test_scan_ai.py b/tests/harness/test_scan_ai.py new file mode 100644 index 000000000..e53825f6a --- /dev/null +++ b/tests/harness/test_scan_ai.py @@ -0,0 +1,1052 @@ +"""The AI step of the agent scan: one paid turn, what it ends as, and the tools the model reads with. + +No test calls a model. A fake backend plays a script: what the model says, which tools it calls +(through the real handlers), and how its turn ends. Every test that waits on a stage runs under +its own bound, so a missing bound in the code fails the test instead of hanging the run. +""" + +from __future__ import annotations + +import asyncio +import logging +import os +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from fi.alk.harness import scan_ai, session, spend +from fi.alk.harness.backends import Call, ModelReply, Say, StageDone, ToolReturned +from fi.alk.harness.config import load_skill + +GEMINI = "gemini-fake-model" +SUBMIT = "mcp__scan__submit_agents" +READ = "mcp__scan_files__read" +GLOB = "mcp__scan_files__glob" +GREP = "mcp__scan_files__grep" +AGENT = ("services/support", "services/support/agent.py") +KEPT = ("call", SUBMIT, {"agents": [{"folder": AGENT[0], "entry": AGENT[1]}]}) +FINISHED = ("done", StageDone(outcome="success", turns=3, cost_usd=0.01, models={GEMINI})) +# ADK's whole message, as the Vertex-Gemini backend passes it on. +CALL_LIMIT_MESSAGE = "Max number of llm calls limit of `40` exceeded" +TEST_BOUND = 15 + +# Read before any test shortens them. +_BOUNDS = { + name: getattr(scan_ai, name) + for name in ( + "OPEN_SECONDS", + "INTERRUPT_SECONDS", + "SPEND_WAIT_SECONDS", + "EXIT_SECONDS", + "TOOL_SECONDS", + "IDLE_SECONDS", + "MAX_TURNS", + ) +} + + +@pytest.fixture(autouse=True) +def _clean_ledger(monkeypatch): + retries = session.STAGE_IDLE_RETRIES + monkeypatch.delenv("ALK_SPEND_JOURNAL", raising=False) + monkeypatch.setattr(spend, "_path", None) + monkeypatch.setenv("ALK_HARNESS_MODEL", GEMINI) + yield + session.STAGE_IDLE_RETRIES = retries + spend._stages.clear() + + +@pytest.fixture +def quick(monkeypatch): + for name, seconds in ( + ("OPEN_SECONDS", 0.3), + ("INTERRUPT_SECONDS", 0.2), + ("SPEND_WAIT_SECONDS", 0.3), + ("CANCEL_SECONDS", 0.2), + ("EXIT_SECONDS", 0.2), + ): + monkeypatch.setattr(scan_ai, name, seconds) + + +def _write(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _repo(tmp_path: Path) -> Path: + source = tmp_path / "repo" + _write(source / AGENT[1], "from openai import OpenAI\nclient = OpenAI()\n") + _write(source / "services/support/pyproject.toml", "[project]\nname = 'support'\n") + _write(source / "main.py", "print('hello')\n") + _write(source / "tests/test_agent.py", "def test_nothing():\n pass\n") + return source + + +def _said(result: dict[str, Any]) -> str: + return "".join(part.get("text", "") for part in result.get("content", [])) + + +class _Backend: + """Sessions that play ``steps`` instead of calling a model, and count what they were asked.""" + + name = "fake" + default_model = GEMINI + + def __init__(self, *steps: tuple, start: str = "", stop: str = "", interrupt: str = ""): + self.steps = steps + self.start = start + self.stop = stop + self.interrupt = interrupt + self.specs: list[Any] = [] + self.sent: list[str] = [] + self.results: list[dict[str, Any]] = [] + self.interrupts = 0 + self.stops = 0 + + def create(self, spec): + self.specs.append(spec) + return _Session(self, spec) + + def can_drive(self, model: str) -> bool: + return True + + +class _Session: + def __init__(self, backend: _Backend, spec) -> None: + self.backend = backend + self.spec = spec + self.interrupted = asyncio.Event() + + async def start(self) -> None: + if self.backend.start == "hang": + await asyncio.Event().wait() + if self.backend.start == "raise": + raise RuntimeError("the provider refused the session") + + async def stop(self) -> None: + self.backend.stops += 1 + if self.backend.stop == "hang": + await asyncio.Event().wait() + + async def send(self, message: str) -> None: + self.backend.sent.append(message) + + async def interrupt(self) -> bool: + self.backend.interrupts += 1 + if self.backend.interrupt == "hang": + await asyncio.Event().wait() + self.interrupted.set() + return True + + async def resume(self, invocation_id: str) -> None: + raise RuntimeError("not supported") + + def _handler(self, qualified: str): + _, server, name = qualified.split("__") + return next( + spec.handler for spec in self.spec.servers[server].tools if spec.name == name + ) + + async def replies(self): + for step in self.backend.steps: + kind = step[0] + if kind == "say": + yield ModelReply(parts=[Say(text=step[1])], model=GEMINI) + elif kind == "call": + call_id = f"call-{len(self.backend.results)}" + yield ModelReply(parts=[Call(id=call_id, name=step[1], arguments=step[2])]) + result = await self._handler(step[1])(step[2]) + self.backend.results.append(result) + yield ToolReturned( + id=call_id, text=_said(result), is_error=bool(result.get("is_error")) + ) + elif kind == "done": + yield step[1] + elif kind == "quiet": + await asyncio.Event().wait() + elif kind == "until_interrupt": + await self.interrupted.wait() + elif kind == "raise": + raise step[1] + + +def _scan(monkeypatch, source: Path, backend: _Backend, *, seconds: float = 5.0, + hints: str = "3 Python files outside side folders", ends: list | None = None): + monkeypatch.setattr(session, "resolve", lambda *_args, **_kwargs: backend) + noted = ends if ends is not None else [] + return asyncio.run( + asyncio.wait_for( + scan_ai.run_ai_step( + source, hints=hints, seconds=seconds, on_turn_end=lambda: noted.append(1) + ), + TEST_BOUND, + ) + ) + + +@dataclass +class _Case: + steps: tuple + kind: str + retryable: bool = False + complete: bool = True + ends: int = 1 + seconds: float = 5.0 + interrupt: str = "" + idle: float | None = None + pairs: tuple = field(default_factory=tuple) + + +def _done(**facts: Any) -> tuple: + facts.setdefault("models", {GEMINI}) + return ("done", StageDone(**facts)) + + +_OUTCOMES = { + "pairs": _Case((KEPT, FINISHED), "pairs", pairs=(AGENT,)), + "pairs_then_a_forbidden_model": _Case( + (KEPT, _done(models={"claude-sonnet-4-6"})), "pairs", pairs=(AGENT,) + ), + "forbidden": _Case((_done(models={"claude-sonnet-4-6"}),), "forbidden"), + "timed_out_quiet": _Case((("quiet",),), "timed_out", complete=False, ends=0, seconds=0.3), + "timed_out_bill_after_the_interrupt": _Case( + (("until_interrupt",), FINISHED), "timed_out", complete=False, seconds=0.3 + ), + "interrupt_never_returns": _Case( + (("quiet",),), "timed_out", complete=False, ends=0, seconds=0.3, interrupt="hang" + ), + "idle_stream": _Case((("quiet",),), "model_error", True, complete=False, ends=0, idle=0.2), + "say_raises": _Case( + (("raise", RuntimeError("stream broke")),), "model_error", True, complete=False, ends=0 + ), + "status_400": _Case( + (_done(outcome="failed", is_error=True, api_error_status=400),), "model_error", False + ), + "status_408": _Case( + (_done(outcome="failed", is_error=True, api_error_status=408),), "model_error", True + ), + "status_409": _Case( + (_done(outcome="failed", is_error=True, api_error_status=409),), "model_error", True + ), + "status_429": _Case( + (_done(outcome="failed", is_error=True, api_error_status=429),), "model_error", True + ), + "status_500": _Case( + (_done(outcome="failed", is_error=True, api_error_status=500),), "model_error", True + ), + "is_error_without_status": _Case( + (_done(outcome="failed", is_error=True, turns=3),), "model_error", True + ), + "is_error_without_status_on_the_last_turn": _Case( + ( + _done( + outcome="failed", + is_error=True, + api_error_status=None, + turns=40, + errors=["The connection was reset by the server."], + ), + ), + "model_error", + True, + ), + "call_limit_words_inside_other_text": _Case( + ( + _done( + outcome="failed", + is_error=True, + api_error_status=None, + turns=7, + errors=[f"Retry failed: {CALL_LIMIT_MESSAGE} (upstream)"], + ), + ), + "model_error", + True, + ), + "call_limit_words_without_a_number": _Case( + ( + _done( + outcome="failed", + is_error=True, + api_error_status=None, + turns=7, + errors=["Max number of llm calls limit of `` exceeded"], + ), + ), + "model_error", + True, + ), + "claude_turn_limit": _Case( + (_done(outcome="error_max_turns", is_error=True, turns=40),), "no_answer" + ), + "gemini_max_turns": _Case( + (_done(outcome="max_turns", is_error=True, turns=12),), "no_answer" + ), + "gemini_call_limit": _Case( + ( + _done( + outcome="failed", + is_error=True, + api_error_status=None, + turns=40, + errors=[CALL_LIMIT_MESSAGE], + ), + ), + "no_answer", + ), + "gemini_call_limit_in_adks_words": _Case( + ( + _done( + outcome="failed", + is_error=True, + api_error_status=None, + turns=7, + errors=[CALL_LIMIT_MESSAGE], + ), + ), + "no_answer", + ), + "empty": _Case((("call", SUBMIT, {"agents": []}), FINISHED), "empty"), + "every_pair_refused": _Case( + (("call", SUBMIT, {"agents": [{"folder": "", "entry": "missing.py"}]}), FINISHED), + "empty", + ), + "no_answer": _Case((("say", "I looked around."), FINISHED), "no_answer"), +} + + +@pytest.mark.parametrize("name", sorted(_OUTCOMES)) +def test_each_ai_outcome_gives_its_result(tmp_path, monkeypatch, quick, name): + case = _OUTCOMES[name] + if case.idle is not None: + monkeypatch.setattr(scan_ai, "IDLE_SECONDS", case.idle) + monkeypatch.setattr(session, "STAGE_IDLE_RETRIES", 1) + backend = _Backend(*case.steps, interrupt=case.interrupt) + ends: list = [] + + outcome = _scan(monkeypatch, _repo(tmp_path), backend, seconds=case.seconds, ends=ends) + + assert outcome.kind == case.kind + assert outcome.pairs == case.pairs + assert outcome.complete is case.complete + assert outcome.retryable is case.retryable + assert len(ends) == case.ends + # The scan's stage turns replay off for itself, never for the whole process. + assert session.STAGE_IDLE_RETRIES == 1 + assert len(backend.sent) == 1 + if case.kind == "timed_out": + assert backend.interrupts == 1 + + +def test_a_gemini_call_limit_is_no_answer(tmp_path, monkeypatch): + """The real Vertex-Gemini backend and ADK, with only the model scripted: it never submits.""" + pytest.importorskip("google.adk") + from google.adk.models.base_llm import BaseLlm + from google.adk.models.llm_response import LlmResponse + from google.genai import types + + from fi.alk.harness.backends import vertex_gemini + + calls: list[int] = [] + + class LoopingLlm(BaseLlm): + async def generate_content_async(self, llm_request, stream=False): + calls.append(1) + yield LlmResponse( + content=types.Content( + role="model", + parts=[ + types.Part( + function_call=types.FunctionCall( + name=GLOB, args={"pattern": "*.py"}, id=f"c{len(calls)}" + ) + ) + ], + ), + usage_metadata=types.GenerateContentResponseUsageMetadata( + prompt_token_count=1000, candidates_token_count=10 + ), + ) + + # The backend writes these three itself; set here so they are put back afterwards. + for name, value in ( + ("GOOGLE_CLOUD_PROJECT", "fake-project"), + ("GOOGLE_CLOUD_LOCATION", "global"), + ("GOOGLE_GENAI_USE_VERTEXAI", "TRUE"), + ("ALK_HARNESS_MODEL", "gemini-2.5-flash"), + ): + monkeypatch.setenv(name, value) + monkeypatch.delenv("AGENTCC_API_KEY", raising=False) + monkeypatch.setattr(vertex_gemini, "COMPACT_ABOVE_TOKENS", 0) + monkeypatch.setattr(vertex_gemini, "_llm", lambda model: LoopingLlm(model=model)) + for name, seconds in ( + ("INTERRUPT_SECONDS", 0.2), + ("SPEND_WAIT_SECONDS", 0.3), + ("CANCEL_SECONDS", 0.2), + ("EXIT_SECONDS", 1.0), + ("OPEN_SECONDS", 5.0), + ): + monkeypatch.setattr(scan_ai, name, seconds) + ends: list = [] + + outcome = _scan( + monkeypatch, + _repo(tmp_path), + vertex_gemini.VertexGeminiBackend(), + seconds=8.0, + ends=ends, + ) + + assert outcome.kind == "no_answer" + assert outcome.complete is True + assert len(ends) == 1 + assert 1 <= len(calls) <= scan_ai.MAX_TURNS + 1 + assert "llm calls" not in outcome.message.lower() + + +def test_an_idle_ai_stream_is_not_replayed(tmp_path, monkeypatch, quick): + monkeypatch.setattr(session, "STAGE_IDLE_RETRIES", 1) + monkeypatch.setattr(scan_ai, "IDLE_SECONDS", 0.2) + backend = _Backend(("quiet",)) + + outcome = _scan(monkeypatch, _repo(tmp_path), backend) + + assert outcome.kind == "model_error" + assert outcome.retryable is True + assert len(backend.specs) == 1 + assert len(backend.sent) == 1 + # The scan's stage turns replay off for itself, never for the whole process. + assert session.STAGE_IDLE_RETRIES == 1 + + +def test_pairs_submitted_before_an_idle_stream_are_kept(tmp_path, monkeypatch, quick): + monkeypatch.setattr(scan_ai, "IDLE_SECONDS", 0.2) + backend = _Backend(KEPT, ("quiet",)) + + outcome = _scan(monkeypatch, _repo(tmp_path), backend) + + assert outcome.kind == "pairs" + assert outcome.pairs == (AGENT,) + assert outcome.complete is False + + +# A model that ends without submitting is the turn a reminder would be tempted to follow. +@pytest.mark.parametrize( + ("steps", "kind"), + [((("say", "Looking."), KEPT, FINISHED), "pairs"), ((("say", "Looking."), FINISHED), "no_answer")], + ids=["submitted", "ended_without_submitting"], +) +def test_the_stage_says_exactly_once(tmp_path, monkeypatch, quick, steps, kind): + backend = _Backend(*steps) + ends: list = [] + hints = "3 Python files outside side folders" + + outcome = _scan(monkeypatch, _repo(tmp_path), backend, hints=hints, ends=ends) + + assert outcome.kind == kind + assert len(backend.specs) == 1 + assert backend.sent == [scan_ai._opening(hints)] + assert len(ends) == 1 + + +def test_complete_is_false_after_the_stop_sequence(tmp_path, monkeypatch, quick): + assert _BOUNDS["INTERRUPT_SECONDS"] == 10 + assert _BOUNDS["SPEND_WAIT_SECONDS"] == 60 + backend = _Backend(KEPT, ("until_interrupt",), FINISHED) + ends: list = [] + + outcome = _scan(monkeypatch, _repo(tmp_path), backend, seconds=0.3, ends=ends) + + assert outcome.kind == "pairs" + assert backend.interrupts == 1 + # The bill arrived during the wait, and still the turn may be missing a call in flight. + assert len(ends) == 1 + assert outcome.complete is False + + +@pytest.mark.parametrize("start", ["hang", "raise"]) +def test_a_stage_that_will_not_open_is_a_model_error(tmp_path, monkeypatch, quick, start): + assert _BOUNDS["OPEN_SECONDS"] == 30 + backend = _Backend(KEPT, FINISHED, start=start) + + outcome = _scan(monkeypatch, _repo(tmp_path), backend) + + assert outcome.kind == "model_error" + assert outcome.retryable is True + assert outcome.complete is False + assert backend.sent == [] + + +def test_a_stage_that_will_not_stop_is_left_after_ten_seconds(tmp_path, monkeypatch, quick): + assert _BOUNDS["EXIT_SECONDS"] == 10 + backend = _Backend(KEPT, FINISHED, stop="hang") + + outcome = _scan(monkeypatch, _repo(tmp_path), backend) + + assert outcome.kind == "pairs" + assert outcome.complete is True + assert backend.stops == 1 + + +def test_the_opening_carries_no_path_and_the_outcome_no_model_text(tmp_path, monkeypatch, quick): + source = _repo(tmp_path) + hints = "2 Python files outside side folders; 1 candidate in side folders" + model_text = "MODEL-TEXT services/support/agent.py says sk-fake-123" + assert "/" not in scan_ai._opening("") + + failed = _Backend( + ("say", model_text), + _done(outcome="failed", is_error=True, turns=2, errors=[model_text]), + ) + outcome = _scan(monkeypatch, source, failed, hints=hints) + assert outcome.kind == "model_error" + assert failed.sent == [scan_ai._opening(hints)] + assert failed.sent[0].endswith(hints) + for path in (str(source), str(tmp_path), "services/", "tests/", "agent.py"): + assert path not in failed.sent[0] + + raised = _Backend(("raise", RuntimeError(model_text)),) + raised_outcome = _scan(monkeypatch, source, raised, hints=hints) + assert raised_outcome.message == "the model step failed (RuntimeError)" + + answered = _Backend(("say", model_text), KEPT, FINISHED) + kept = _scan(monkeypatch, source, answered, hints=hints) + assert kept.pairs == (AGENT,) + + for one in (outcome, raised_outcome, kept): + assert "MODEL-TEXT" not in one.message + assert "sk-fake-123" not in one.message + + +def test_the_stage_has_no_built_in_tools(tmp_path, monkeypatch, quick): + source = _repo(tmp_path) + backend = _Backend(FINISHED) + + _scan(monkeypatch, source, backend) + + spec = backend.specs[0] + assert spec.builtins == () + assert spec.ask is None + assert spec.gated is True + assert spec.thinking is False + assert spec.max_turns == 40 + assert spec.idle_timeout_seconds == 90 + assert spec.cwd == str(source) + assert spec.model == GEMINI + assert spec.system_prompt == load_skill("find-agents") + assert set(spec.servers) == {"scan_files", "scan"} + assert sorted(spec.granted()) == sorted([READ, GLOB, GREP, SUBMIT]) + for name in (READ, GLOB, GREP, SUBMIT): + assert name in spec.system_prompt + + +def test_the_skills_description_says_when_the_model_is_asked(): + # The model reads the whole file, its description included. + skill = load_skill("find-agents", preamble=False) + description = yaml.safe_load(skill.split("---\n")[1])["description"] + + assert "came back empty" not in skill + assert "found no agent outside test and example folders" in description + assert "apart from any files that only call a model" in description + assert "TypeScript or JavaScript" in description + assert skill in load_skill("find-agents") + + +# -- the tools --------------------------------------------------------------------------------- + + +def _tools(root: Path) -> dict[str, Any]: + return {spec.name: spec.handler for spec in scan_ai.rooted_file_tools(root).tools} + + +def _call(handler, arguments: Any) -> dict[str, Any]: + return asyncio.run(asyncio.wait_for(handler(arguments), TEST_BOUND)) + + +def _listed(text: str) -> set[str]: + """The paths a glob or grep result names.""" + return {line.split(":", 1)[0] for line in text.splitlines() if line} + + +def test_the_ai_cannot_read_outside_the_source(tmp_path): + outside = _write(tmp_path / "outside/creds.txt", "outside-needle\n") + source = tmp_path / "repo" + _write(source / "inside.py", "inside-needle\n") + os.symlink(outside.parent, source / "out_link") + os.symlink(outside, source / "file_link.py") + tools = _tools(source) + + for path in ( + "../outside/creds.txt", + str(outside), + "out_link/creds.txt", + "file_link.py", + "inside.py/../../outside/creds.txt", + ): + result = _call(tools["read"], {"path": path}) + assert result.get("is_error"), path + assert "outside-needle" not in _said(result) + for path in ("../outside", "/", str(tmp_path), "out_link"): + assert _call(tools["glob"], {"pattern": "*", "path": path}).get("is_error"), path + assert _call(tools["grep"], {"text": "needle", "path": path}).get("is_error"), path + + assert _listed(_said(_call(tools["glob"], {"pattern": "*"}))) == {"inside.py"} + found = _said(_call(tools["grep"], {"text": "needle"})) + assert "outside-needle" not in found + assert _listed(found) == {"inside.py"} + assert "inside-needle" in _said(_call(tools["read"], {"path": str(source / "inside.py")})) + + +_HIDDEN = ( + ".env", + "config/service_account.json", + "config/token.json", + "prod.env", + "app.env.local", + "secrets.toml", + "credentials.yml", + "id_rsa", + "terraform.tfvars", + "secrets/keys.py", +) + + +def test_grep_and_glob_never_show_a_secret_file(tmp_path): + source = tmp_path / "repo" + _write(source / "agent.py", "needle = 1\n") + _write(source / "config/settings.py", "needle = 2\n") + for name in _HIDDEN: + _write(source / name, "needle API_KEY=sk-fake-123\n") + tools = _tools(source) + + for path in (None, ".", "config", str(source), str(source / "config")): + arguments = {} if path is None else {"path": path} + for grep in ({"text": "needle"}, {"text": "NEEDLE", "ignore_case": True}): + found = _said(_call(tools["grep"], {**grep, **arguments})) + assert "sk-fake-123" not in found + assert not _listed(found) & set(_HIDDEN), (path, found) + assert _listed(found) <= {"agent.py", "config/settings.py"} + for pattern in ("*", "**", "*.env", "*.json", "id_*", "*secret*"): + listed = _said(_call(tools["glob"], {"pattern": pattern, **arguments})) + assert not _listed(listed) & set(_HIDDEN), (path, pattern, listed) + assert "agent.py" in _listed(_said(_call(tools["grep"], {"text": "needle"}))) + for name in _HIDDEN: + result = _call(tools["read"], {"path": name}) + assert result.get("is_error"), name + assert "sk-fake-123" not in _said(result) + + +def test_the_tools_skip_fifos_and_big_files_and_hard_links(tmp_path): + source = tmp_path / "repo" + _write(source / "agent.py", "needle\n") + os.mkfifo(source / "pipe.py") + _write(source / "big.py", "needle\n" * 300_000) + _write(source / "linked.py", "needle\n") + os.link(source / "linked.py", source / "also.py") + os.link(_write(tmp_path / "outside.py", "needle\n"), source / "hard.py") + _write(source / "app.min.js", "needle\n") + _write(source / "long.py", "needle" + "x" * 70_000 + "\nneedle\n") + tools = _tools(source) + + assert _listed(_said(_call(tools["glob"], {"pattern": "*"}))) == {"agent.py", "long.py"} + found = _said(_call(tools["grep"], {"text": "needle"})) + assert found.splitlines() == ["agent.py:1:needle", "long.py:2:needle"] + for name in ("pipe.py", "big.py", "linked.py", "also.py", "hard.py"): + assert _call(tools["read"], {"path": name}).get("is_error"), name + assert _call(tools["grep"], {"text": "needle", "path": name}).get("is_error"), name + + +def test_a_slow_tool_call_ends_with_an_error(tmp_path, monkeypatch): + assert _BOUNDS["TOOL_SECONDS"] == 20 + source = _repo(tmp_path) + monkeypatch.setattr(scan_ai, "TOOL_SECONDS", 0.2) + monkeypatch.setattr(scan_ai, "_grep_files", lambda root, arguments: time.sleep(3) or "late") + tools = _tools(source) + + began = time.monotonic() + result = _call(tools["grep"], {"text": "needle"}) + + assert time.monotonic() - began < 2 + assert result.get("is_error") + assert "late" not in _said(result) + + +def test_no_tool_raises(tmp_path, monkeypatch): + source = _repo(tmp_path) + tools = _tools(source) + calls = [ + ("read", {"path": "main\x00.py"}), + ("read", {"path": "main.py", "offset": "x"}), + ("read", {"path": "main.py", "limit": [1]}), + ("read", {"path": "/".join(["a"] * 3_000)}), + ("read", {"path": "a" * 5_000}), + ("read", {"path": 7}), + ("read", {}), + ("read", None), + ("glob", {"pattern": "x" * 300}), + ("glob", {"pattern": None}), + ("glob", {"pattern": "*", "path": "\x00"}), + ("grep", {"text": ""}), + ("grep", {"text": "x" * 201}), + ("grep", {"text": "a", "ignore_case": "yes"}), + ("grep", {"text": "a", "glob": 5}), + ("grep", {"text": "a", "path": "missing/folder"}), + ] + for name, arguments in calls: + assert _call(tools[name], arguments).get("is_error"), (name, arguments) + + def broken(*_args): + raise RuntimeError("MODEL-TEXT from inside a worker") + + for worker in ("_read_lines", "_glob_files", "_grep_files", "_checked_pairs"): + monkeypatch.setattr(scan_ai, worker, broken) + for name, arguments in ( + ("read", {"path": "main.py"}), + ("glob", {"pattern": "*"}), + ("grep", {"text": "a"}), + ): + result = _call(tools[name], arguments) + assert result.get("is_error") + assert _said(result) == scan_ai._FAILED + submit = scan_ai._submit_server(source, scan_ai._Answer()).tools[0].handler + submitted = _call(submit, {"agents": [{"folder": "", "entry": "main.py"}]}) + assert submitted.get("is_error") + assert _said(submitted) == scan_ai._FAILED + + +def test_tool_paths_are_relative(tmp_path): + source = tmp_path / "repo" + _write(source / "pkg/agent.py", "needle\n") + tools = _tools(source) + + for path in (None, "pkg", str(source / "pkg")): + arguments = {} if path is None else {"path": path} + listed = _said(_call(tools["glob"], {"pattern": "*.py", **arguments})) + found = _said(_call(tools["grep"], {"text": "needle", **arguments})) + assert listed == "pkg/agent.py" + assert found == "pkg/agent.py:1:needle" + for text in (listed, found): + assert str(tmp_path) not in text + assert _said(_call(tools["read"], {"path": "pkg/agent.py"})) == "1\tneedle" + + +def test_a_capped_search_says_it_stopped_early(tmp_path): + source = tmp_path / "repo" + for index in range(2_100): + _write(source / f"many/f{index:04}.txt", "") + tools = _tools(source) + + for name, arguments in ( + ("glob", {"pattern": "*.nothing"}), + ("grep", {"text": "nothing-here"}), + ): + said = _said(_call(tools[name], arguments)) + assert said == scan_ai.STOPPED_EARLY, (name, said) + + full = tmp_path / "full" + _write(full / "hits.py", "".join(f"needle {index}\n" for index in range(2_000))) + _write(full / "wide.py", "".join(f"{'y' * 100}\n" for _ in range(400))) + full_tools = _tools(full) + for name, arguments in ( + ("grep", {"text": "needle"}), + ("read", {"path": "wide.py"}), + ): + said = _said(_call(full_tools[name], arguments)) + assert said.endswith(scan_ai.STOPPED_EARLY), name + assert len(said) <= 8_000, name + + +def test_one_long_hit_does_not_hide_the_others(tmp_path): + source = tmp_path / "repo" + _write(source / "a.py", "needle" + "x" * 20_000 + "\n") + for name in ("b.py", "c.py", "d.py", "e.py"): + _write(source / name, "needle\n") + tools = _tools(source) + + found = _said(_call(tools["grep"], {"text": "needle"})) + + assert _listed(found) == {"a.py", "b.py", "c.py", "d.py", "e.py"} + assert scan_ai.STOPPED_EARLY not in found + for line in found.splitlines(): + assert len(line.split(":", 2)[2]) <= 300 + + +# -- the answer -------------------------------------------------------------------------------- + + +def _submit(source: Path, agents: Any) -> tuple[str, Any]: + answer = scan_ai._Answer() + handler = scan_ai._submit_server(source, answer).tools[0].handler + return _said(_call(handler, {"agents": agents})), answer + + +def test_the_models_paths_are_checked(tmp_path): + source = _repo(tmp_path) + _write(source / "node_modules/pkg/agent.js", "x\n") + _write(source / "build/agent.py", "x\n") + _write(source / "services/other/run.py", "x\n") + evil = _write(tmp_path / "outside/evil.py", "x\n") + os.symlink(evil, source / "services/support/evil.py") + os.symlink(evil.parent, source / "linked") + + for folder, entry, kept in ( + ("", "main.py", ("", "main.py")), + ("", AGENT[1], ("", AGENT[1])), + (AGENT[0], AGENT[1], AGENT), + ("./" + AGENT[0], "./" + AGENT[1], AGENT), + (f"{source}/{AGENT[0]}", f"{source}/{AGENT[1]}", AGENT), + (str(source), f"{source}/main.py", ("", "main.py")), + ): + said, answer = _submit(source, [{"folder": folder, "entry": entry}]) + assert said.startswith("Recorded 1; 0 refused."), (folder, entry, said) + assert answer.pairs == (kept,) + + for agent in ( + {"folder": "services/other", "entry": AGENT[1]}, + {"folder": "services/sup", "entry": AGENT[1]}, + {"folder": "", "entry": "../outside/evil.py"}, + {"folder": "", "entry": str(evil)}, + {"folder": AGENT[0], "entry": "services/support/evil.py"}, + {"folder": "linked", "entry": "linked/evil.py"}, + {"folder": "", "entry": AGENT[0]}, + {"folder": "", "entry": "missing.py"}, + {"folder": "", "entry": "node_modules/pkg/agent.js"}, + {"folder": "", "entry": "build/agent.py"}, + {"folder": "", "entry": "main.py\n"}, + {"folder": "", "entry": ""}, + {"folder": "../", "entry": "main.py"}, + {"folder": 1, "entry": "main.py"}, + "main.py", + ): + said, answer = _submit(source, [agent]) + assert said.startswith("Recorded 0; 1 refused."), (agent, said) + assert answer.pairs == () + assert answer.called is True + + said, answer = _submit(source, [{"folder": "", "entry": "main.py"}] * 2) + assert said.startswith("Recorded 1; 0 refused.") + assert answer.pairs == (("", "main.py"),) + + said, answer = _submit(source, [{"folder": "", "entry": "main.py"}] * 51) + assert answer.pairs == () + assert answer.called is False + + +def test_the_first_kept_submit_is_final_and_the_tools_stop(tmp_path, monkeypatch, quick): + backend = _Backend( + ("call", SUBMIT, {"agents": [{"folder": "", "entry": "missing.py"}]}), + KEPT, + ("call", SUBMIT, {"agents": [{"folder": "", "entry": "main.py"}]}), + ("call", READ, {"path": "main.py"}), + ("call", GLOB, {"pattern": "*"}), + ("call", GREP, {"text": "print"}), + FINISHED, + ) + + outcome = _scan(monkeypatch, _repo(tmp_path), backend) + + said = [_said(result) for result in backend.results] + assert said[0].startswith("Recorded 0; 1 refused.") + assert said[1] == "Recorded 1; 0 refused. Your work is done: end your turn now." + assert said[2] == "Already submitted." + assert said[3:] == ["Already submitted: end your turn."] * 3 + assert outcome.kind == "pairs" + assert outcome.pairs == (AGENT,) + assert outcome.refused == 0 + + +def test_a_refused_pair_in_the_kept_submit_is_counted(tmp_path, monkeypatch, quick): + agents = [ + {"folder": AGENT[0], "entry": AGENT[1]}, + {"folder": "", "entry": "missing.py"}, + {"folder": "services/other", "entry": AGENT[1]}, + ] + backend = _Backend(("call", SUBMIT, {"agents": agents}), FINISHED) + + outcome = _scan(monkeypatch, _repo(tmp_path), backend) + + assert _said(backend.results[0]).startswith("Recorded 1; 2 refused.") + assert outcome.kind == "pairs" + assert outcome.pairs == (AGENT,) + assert outcome.refused == 2 + + +def test_an_entry_reached_through_a_link_out_of_the_source_is_refused(tmp_path): + source = _repo(tmp_path) + _write(tmp_path / "outside/other.py", "x\n") + # A link in the middle of the path: the entry itself is a regular file, and its folder is + # inside the source. + os.symlink(tmp_path / "outside", source / "services/support/ext") + + for folder in (AGENT[0], ""): + said, answer = _submit(source, [{"folder": folder, "entry": f"{AGENT[0]}/ext/other.py"}]) + assert said.startswith("Recorded 0; 1 refused."), (folder, said) + assert answer.pairs == () + + +def test_the_tools_work_when_finished_is_none(tmp_path): + source = _repo(tmp_path) + tools = { + spec.name: spec.handler + for spec in scan_ai.rooted_file_tools(source, finished=None).tools + } + + result = _call(tools["read"], {"path": "main.py"}) + + assert not result.get("is_error"), _said(result) + assert "hello" in _said(result) + assert not _call(tools["glob"], {"pattern": "*.py"}).get("is_error") + + +def test_two_submits_at_once_keep_exactly_one_set(tmp_path, monkeypatch): + source = _repo(tmp_path) + + async def checked_after_a_yield(work, *args): + # A thread's answer comes back on a later turn of the loop; yielding once here makes + # the two calls interleave the same way on every run. + await asyncio.sleep(0) + return work(*args) + + monkeypatch.setattr(scan_ai, "_in_thread", checked_after_a_yield) + answer = scan_ai._Answer() + handler = scan_ai._submit_server(source, answer).tools[0].handler + first = [{"folder": AGENT[0], "entry": AGENT[1]}] + second = [{"folder": "", "entry": "main.py"}] + + async def both() -> list[dict[str, Any]]: + return list(await asyncio.gather(handler({"agents": first}), handler({"agents": second}))) + + said = [_said(result) for result in asyncio.run(asyncio.wait_for(both(), TEST_BOUND))] + + recorded = "Recorded 1; 0 refused. Your work is done: end your turn now." + assert sorted(said) == sorted([recorded, scan_ai.SUBMIT_AFTER_SUBMIT]) + assert answer.pairs == ((AGENT,), (("", "main.py"),))[said.index(recorded)] + + +# -- a path the platform can't carry (the vercel/ai-chatbot run, 6 Nov) ----------------------- + +ROUTE = "app/(chat)/api/chat/route.ts" + + +def _route_repo(tmp_path: Path) -> Path: + source = _repo(tmp_path) + _write(source / ROUTE, "export async function POST() {}\n") + return source + + +def test_a_real_file_the_path_rule_refuses_is_counted_once_and_named( + tmp_path, monkeypatch, quick +): + backend = _Backend( + ("call", SUBMIT, {"agents": [{"folder": "", "entry": ROUTE}]}), + ("call", SUBMIT, {"agents": [{"folder": "app", "entry": "./" + ROUTE}]}), + FINISHED, + ) + + outcome = _scan(monkeypatch, _route_repo(tmp_path), backend) + + said = [_said(result) for result in backend.results] + for line in said: + assert line.startswith("Recorded 0; 1 refused. 1 name a file whose path the platform") + assert "don't submit them again" in line + assert outcome.kind == "empty" + assert outcome.uncarried == 1 + + +@pytest.mark.parametrize( + "entry", + ["app/(chat)/missing.ts", "../outside/(x).ts", "/etc/(x).ts", "node_modules/(x)/a.ts"], +) +def test_a_missing_or_outside_file_is_not_counted_as_uncarried( + tmp_path, monkeypatch, quick, entry +): + source = _route_repo(tmp_path) + _write(source / "node_modules/(x)/a.ts", "x\n") + backend = _Backend(("call", SUBMIT, {"agents": [{"folder": "", "entry": entry}]}), FINISHED) + + outcome = _scan(monkeypatch, source, backend) + + assert "can't carry" not in _said(backend.results[0]) + assert outcome.kind == "empty" + assert outcome.uncarried == 0 + + +def test_an_uncarried_file_before_a_kept_submit_is_still_counted(tmp_path, monkeypatch, quick): + backend = _Backend( + ("call", SUBMIT, {"agents": [{"folder": "", "entry": ROUTE}]}), + KEPT, + FINISHED, + ) + + outcome = _scan(monkeypatch, _route_repo(tmp_path), backend) + + assert outcome.kind == "pairs" + assert outcome.pairs == (AGENT,) + assert outcome.refused == 0 + assert outcome.uncarried == 1 + + +def test_glob_reads_a_pattern_as_a_shell_would(tmp_path): + source = _route_repo(tmp_path) + _write(source / "package.json", "{}\n") + _write(source / "web/package.json", "{}\n") + _write(source / "web/src/page.tsx", "x\n") + tools = _tools(source) + + def globbed(pattern: str, **extra: str) -> set[str]: + return _listed(_said(_call(tools["glob"], {"pattern": pattern, **extra}))) + + assert globbed("**/package.json") == {"package.json", "web/package.json"} + assert globbed("**/*.{ts,tsx}") == {ROUTE, "web/src/page.tsx"} + assert globbed("web/**/page.tsx") == {"web/src/page.tsx"} + assert globbed("*.{py,json}") >= {"main.py", "package.json", AGENT[1]} + assert _said(_call(tools["glob"], {"pattern": "{"})) == "No file matches." + assert len(scan_ai._pattern_forms("{a,b}" * 20)) <= scan_ai._MAX_FORMS + found = _said(_call(tools["grep"], {"text": "POST", "glob": "**/*.{ts,tsx}"})) + assert _listed(found) == {ROUTE} + + +def test_a_failed_cost_note_keeps_the_answer_and_logs_its_trace(tmp_path, monkeypatch, quick, caplog): + """A paid answer is never turned into a retry by the cost note, and the note's fault is + logged with its trace so it can be found.""" + backend = _Backend(KEPT, FINISHED) + monkeypatch.setattr(session, "resolve", lambda *_args, **_kwargs: backend) + + def broken_note() -> None: + raise OSError("cost file not writable") + + with caplog.at_level(logging.WARNING, logger=scan_ai.logger.name): + outcome = asyncio.run( + asyncio.wait_for( + scan_ai.run_ai_step( + _repo(tmp_path), hints="x", seconds=5.0, on_turn_end=broken_note + ), + TEST_BOUND, + ) + ) + + assert outcome.kind == "pairs" + assert outcome.pairs == (AGENT,) + assert "the cost note after the turn failed" in caplog.text + assert "Traceback" in caplog.text and "OSError" in caplog.text + + +def test_a_tool_fault_logs_the_workers_own_error(tmp_path, monkeypatch, caplog): + """The model is told only that the call failed; the log carries the worker's real fault.""" + + def broken(root, arguments): + raise KeyError("a fault in the glob worker") + + monkeypatch.setattr(scan_ai, "_glob_files", broken) + tools = _tools(_repo(tmp_path)) + with caplog.at_level(logging.WARNING, logger=scan_ai.logger.name): + said = _said(_call(tools["glob"], {"pattern": "*"})) + + assert said == scan_ai._FAILED + assert "a tool call failed" in caplog.text + assert "KeyError" in caplog.text and "a fault in the glob worker" in caplog.text diff --git a/tests/harness/test_scan_document.py b/tests/harness/test_scan_document.py new file mode 100644 index 000000000..f44aa263e --- /dev/null +++ b/tests/harness/test_scan_document.py @@ -0,0 +1,605 @@ +from __future__ import annotations + +import json +import logging +import os +import re +import stat +from dataclasses import dataclass +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from fi.alk.harness import scan_document as sd +from fi.alk.harness.scan_document import ( + Row, + ScanResultInvalid, + choose_rows, + finish, + root_name, + wire_problems, + write_agents_json, +) + +FIXTURE = Path(__file__).parent / "fixtures" / "agent_scan_e07d56fcd.json" +SHA = "0123456789abcdef0123456789abcdef01234567" +PROBLEM_NAME = re.compile(r"[a-z0-9_.\[\]]+") + + +@dataclass(frozen=True) +class _Candidate: + """The walk's candidate, with the fields the document reads.""" + + entry: str + subfolder: str = "" + sign: str = "framework" + side: bool = False + builds: tuple[str, ...] = ("langgraph",) + build_may_stop: str = "" + partial: bool = False + env_names: tuple[str, ...] = ("OPENAI_API_KEY",) + tools: int = 2 + prompt_files: int = 1 + evidence: tuple[tuple[str, str, int], ...] = () + + +def _framework(name: str = "langgraph", label: str | None = None) -> dict[str, Any]: + return {"name": name, "version": None, "label": label} + + +def _row( + entry: str, + subfolder: str = "", + *, + framework: dict[str, Any] | None = None, + finished: bool = True, + found_by_ai: bool = False, + **candidate: Any, +) -> Row: + return Row( + candidate=_Candidate(entry=entry, subfolder=subfolder, **candidate), + framework=framework if framework is not None else _framework(), + finished=finished, + found_by_ai=found_by_ai, + ) + + +def _finish( + rows: list[Row], + *, + root: str = "shop", + commit_sha: str | None = SHA, + truncated: bool = False, + error: tuple[str, str] | None = None, +) -> tuple[dict[str, Any], bytes, bool]: + body, dropped = finish( + rows=rows, root_name=root, commit_sha=commit_sha, truncated=truncated, error=error + ) + document = json.loads(body) + assert wire_problems(document) == [] + return document, body, dropped + + +def _confidence(*rows: Row) -> list[str]: + document, _, _ = _finish(list(rows)) + return [agent["confidence"] for agent in document["agents"]] + + +def test_found_lists_each_agent_in_the_platform_format() -> None: + graph = _row( + "services/support/graph.py", + "services/support", + env_names=("OPENAI_API_KEY", "SUPPORT_DB_URL"), + tools=3, + evidence=(("builds langgraph", "services/support/graph.py", 12),), + ) + root = _row( + "main.py", + framework=_framework("other", "llama_index"), + builds=("llama_index",), + tools=0, + prompt_files=0, + ) + + document, _, dropped = _finish([root, graph]) + + assert dropped is False + assert list(document) == [ + "schema_version", + "status", + "commit_sha", + "agents", + "truncated", + "error", + ] + assert document == { + "schema_version": "futureagi.agent-scan.v1", + "status": "found", + "commit_sha": SHA, + "agents": [ + { + "name": "support", + "subfolder": "services/support", + "entry": "services/support/graph.py", + "framework": {"name": "langgraph", "version": None, "label": None}, + "confidence": "high", + "evidence": ["Builds with LangGraph in services/support/graph.py, line 12."], + "env_names": ["OPENAI_API_KEY", "SUPPORT_DB_URL"], + "estimates": {"tools": 3, "prompt_files": 1}, + }, + { + "name": "shop", + "subfolder": "", + "entry": "main.py", + "framework": {"name": "other", "version": None, "label": "llama_index"}, + "confidence": "medium", + "evidence": [], + "env_names": ["OPENAI_API_KEY"], + "estimates": {"tools": 0, "prompt_files": 0}, + }, + ], + "truncated": False, + "error": None, + } + assert [list(agent) for agent in document["agents"]] == [list(sd._AGENT_KEYS)] * 2 + assert sd._AGENT_KEYS == ( + "name", + "subfolder", + "entry", + "framework", + "confidence", + "evidence", + "env_names", + "estimates", + ) + + none, _, _ = _finish([]) + assert (none["status"], none["agents"], none["error"]) == ("none", [], None) + error, _, _ = _finish([], error=("scan_timed_out", "The scan ran out of time.")) + assert error["status"] == "error" + assert error["error"] == {"code": "scan_timed_out", "message": "The scan ran out of time."} + assert error["truncated"] is False + + +def test_confidence_rules() -> None: + assert _confidence(_row("a.py", "a")) == ["high"] + assert _confidence(_row("a.py", "a", builds=("langchain",))) == ["medium"] + assert _confidence(_row("a.py", "a", framework=_framework("other", "dspy"))) == ["medium"] + assert _confidence(_row("a.py", "a", framework=_framework("unknown"))) == ["low"] + assert _confidence(_row("a.py", "a", framework=_framework("none"), builds=())) == ["low"] + assert _confidence(_row("a.py", "a", found_by_ai=True)) == ["low"] + assert _confidence(_row("a.py", "a", build_may_stop="graphs")) == ["low"] + assert _confidence(_row("a.py", "a", build_may_stop="agent_py")) == ["low"] + assert _confidence(_row("a.py", "a", partial=True)) == ["low"] + assert _confidence(_row("a.py", "a", finished=False)) == ["low"] + # Two agents in one folder both build cleanly, but a build of that folder can't pick one. + assert _confidence(_row("a/one.py", "a"), _row("a/two.py", "a")) == ["low", "low"] + assert _confidence(_row("a/one.py", "a"), _row("b/two.py", "b")) == ["high", "high"] + + document, _, _ = _finish( + [ + _row("tests/t.py", "tests", side=True), + _row("c/low.py", "c", framework=_framework("unknown")), + _row("b/medium.py", "b", builds=()), + _row("z/high.py", "z"), + _row("a/high.py", "a"), + ] + ) + assert [agent["entry"] for agent in document["agents"]] == [ + "a/high.py", + "z/high.py", + "b/medium.py", + "c/low.py", + "tests/t.py", + ] + + +def test_sharing_a_folder_is_decided_over_the_rows_written() -> None: + shares = sd._TEMPLATES["shares_folder"] + + # The neighbour breaks the platform's path rule, so it isn't written and nothing is shared. + document, _, dropped = _finish([_row("svc/main.py", "svc"), _row("svc/../x.py", "svc")]) + assert dropped is True + assert [agent["entry"] for agent in document["agents"]] == ["svc/main.py"] + assert document["agents"][0]["confidence"] == "high" + assert shares not in document["agents"][0]["evidence"] + + # The neighbour is the fifty-first, cut by the cap. + rows = [_row(f"f{i:02}/main.py", f"f{i:02}") for i in range(50)] + rows.append(_row("f00/tests/test_main.py", "f00", side=True)) + document, _, dropped = _finish(rows) + assert dropped is True + assert len(document["agents"]) == 50 + assert {agent["confidence"] for agent in document["agents"]} == {"high"} + + # The neighbour repeats an entry already listed. + document, _, _ = _finish( + [_row("svc/main.py", "svc"), _row("svc/main.py", "svc", found_by_ai=True)] + ) + assert [(agent["entry"], agent["confidence"]) for agent in document["agents"]] == [ + ("svc/main.py", "high") + ] + + document, _, _ = _finish([_row("svc/main.py", "svc"), _row("svc/other.py", "svc")]) + assert all(shares in agent["evidence"] for agent in document["agents"]) + + +def test_an_uppercase_commit_is_lowercased() -> None: + assert _finish([_row("a.py")], commit_sha=SHA.upper())[0]["commit_sha"] == SHA + assert _finish([_row("a.py")], commit_sha="0123456789ABCdef" * 2 + "01234567")[0][ + "commit_sha" + ] == SHA + assert _finish([_row("a.py")], commit_sha=None)[0]["commit_sha"] is None + assert _finish([_row("a.py")], commit_sha=SHA[:39])[0]["commit_sha"] is None + assert _finish([_row("a.py")], commit_sha="g" * 40)[0]["commit_sha"] is None + assert _finish([_row("a.py")], commit_sha=SHA + "\n")[0]["commit_sha"] is None + + +def test_rows_and_an_error_together_keep_the_rows() -> None: + error = ("scan_timed_out", "The scan ran out of time.") + + document, _, dropped = _finish([_row("a/main.py", "a")], error=error) + assert document["status"] == "found" + assert document["error"] is None + assert document["truncated"] is True + assert [agent["entry"] for agent in document["agents"]] == ["a/main.py"] + assert dropped is False + + # Every row given breaks a rule: nothing is left to list, and the error was already dropped. + document, _, dropped = _finish([_row("a/../main.py", "a")], error=error) + assert (document["status"], document["agents"], document["error"]) == ("none", [], None) + assert document["truncated"] is True + assert dropped is True + + +def test_names_evidence_and_paths_always_fit_the_platform_rules() -> None: + def job(agent_name: Any = None, repository: Any = None) -> Any: + metadata = {} if agent_name is None else {"agent_name": agent_name} + return SimpleNamespace(metadata=metadata, source=SimpleNamespace(repository=repository)) + + assert root_name(job(" Support\u2003Bot\x00 ", "org/shop.git")) == "Support Bot" + assert root_name(job(3, "org/shop.git")) == "shop" + assert root_name(job("\u200b\u3000", "https://github.com/org/shop.git/")) == "shop" + assert root_name(job(None, "shop")) == "shop" + assert root_name(job(None, ".git")) == "agent" + assert root_name(job()) == "agent" + + document, _, _ = _finish( + [ + _row("a/svc/main.py", "a/svc"), + _row("b/svc/app.py", "b/svc"), + _row("c/api/main.py", "c/api"), + _row("d/api/main.py", "d/api"), + _row("main.py"), + _row("server.py"), + ], + root="shop", + ) + assert sorted(agent["name"] for agent in document["agents"]) == [ + "api (c/api/main.py)", + "api (d/api/main.py)", + "shop (main.py)", + "shop (server.py)", + "svc (app.py)", + "svc (main.py)", + ] + assert _finish([_row("main.py")], root="x" * 300)[0]["agents"][0]["name"] == "x" * 255 + assert _finish([_row("main.py")], root="\u200b\x00")[0]["agents"][0]["name"] == "agent" + + long_path = "deep/" * 90 + "graph.py" + facts = tuple(("builds langgraph", f"f{i}/graph.py", i + 1) for i in range(7)) + row = _row( + long_path, + "deep", + evidence=(("builds langgraph", long_path, 4), *facts), + env_names=tuple(f"NAME_{i}" for i in range(60)), + build_may_stop="graphs", + partial=True, + finished=False, + ) + agent = _finish([row])[0]["agents"][0] + assert len(agent["evidence"]) == 5 + assert all(0 < len(sentence) <= 200 for sentence in agent["evidence"]) + assert agent["evidence"][0].startswith("Builds with LangGraph in …/") + assert agent["evidence"][0].endswith("deep/graph.py, line 4.") + assert agent["evidence"][1:4] == [ + sd._TEMPLATES["graphs_may_stop"], + sd._TEMPLATES["not_finished"], + sd._TEMPLATES["partial"], + ] + assert agent["env_names"] == [f"NAME_{i}" for i in range(50)] + + assert sd._LONGEST_TEMPLATE == max(len(template) for template in sd._TEMPLATES.values()) + assert sd._LONGEST_TEMPLATE + sd._PATH_ROOM <= sd.EVIDENCE_MAX + assert sd._PATH_ROOM == 120 + + aside = (("openai_agents_set_aside", "", 0),) + assert sd._TEMPLATES["openai_agents_set_aside"] in _finish( + [_row("a/main.py", "a", framework=_framework("unknown"), evidence=aside)] + )[0]["agents"][0]["evidence"] + assert _finish( + [_row("a/main.py", "a", framework=_framework("openai_agents"), evidence=aside)] + )[0]["agents"][0]["evidence"] == [] + + +def test_clean_name_collapses_every_kind_of_space() -> None: + assert sd._clean_name("a\u2003\u3000b") == "a b" + assert sd._clean_name("\u3000a \t\n b\u2028c\u00a0d\u205f ") == "a b c d" + assert sd._clean_name("a\u200bb\x00c\x1bd\ue000e\ud800f\u202eg\ufeffh") == "abcdefgh" + assert sd._clean_name("\u3000\u2003\u200b") == "" + assert _finish([_row("main.py")], root="Shop\u3000\u2003Bot")[0]["agents"][0]["name"] == ( + "Shop Bot" + ) + + +def test_a_sentence_never_names_a_path_the_path_rule_refuses() -> None: + undecodable = os.fsdecode(b"agent\xff.py") + refused = [ + undecodable, + "../agent.py", + "/srv/agent.py", + "my agent.py", + "a//agent.py", + "agent.py\n", + "x" * 513, + "", + None, + ] + for kind in ("builds langgraph", "graph", "agent_py", "client openai", "project"): + for path in refused: + sentence = sd._sentence(kind, path, 3) + assert sentence is not None + assert sd._UNNAMED_PATH in sentence + if path: + assert path not in sentence + assert "\udcff" not in sentence + sentence.encode("utf-8") + assert len(sentence) <= 200 + + long_path = "d/" * 200 + "agent.py" + sentence = sd._sentence("builds langgraph", long_path, 3) + assert "…/" in sentence and sentence.endswith("/d/agent.py, line 3.") + assert len(sentence) <= 200 + + one_long_part = "a" * 300 + ".py" + sentence = sd._sentence("agent_py", one_long_part, 0) + assert "…" in sentence and len(sentence) <= 200 + + assert sd._sentence("client open ai!", "app.py", 1) == ( + "Creates a client for a model provider in app.py, line 1." + ) + assert sd._sentence("client openai", "app.py", 1) == ( + "Creates a client for openai in app.py, line 1." + ) + assert sd._sentence("builds \x00", "app.py", 1) == ( + "Builds with an agent framework in app.py, line 1." + ) + assert sd._sentence("no_such_fact", "app.py", 1) is None + + +def test_the_file_is_compact_utf8_and_never_fails_on_broken_characters() -> None: + undecodable = os.fsdecode(b"agent\xff.py") + rows = [ + _row("main.py", evidence=(("builds langgraph", undecodable, 2),)), + _row("x" + undecodable, "x" + undecodable), + ] + + document, body, dropped = _finish(rows, root="Café\ud800 Bot") + + assert body == json.dumps(document, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + body.decode("utf-8") + assert "Café Bot".encode("utf-8") in body + assert b"\\u00e9" not in body + assert b"\n" not in body + assert dropped is True + assert [agent["entry"] for agent in document["agents"]] == ["main.py"] + assert document["agents"][0]["evidence"] == [ + "Builds with LangGraph in a file in this folder, line 2." + ] + + +def test_finish_never_writes_more_than_fifty() -> None: + rows = [_row(f"f{i:02}/main.py", f"f{i:02}") for i in range(60)] + + document, _, dropped = _finish(rows) + + assert len(document["agents"]) == 50 + assert document["truncated"] is True + assert dropped is True + assert [agent["entry"] for agent in document["agents"]] == [ + f"f{i:02}/main.py" for i in range(50) + ] + + fifty, _, dropped = _finish(rows[:50]) + assert (len(fifty["agents"]), fifty["truncated"], dropped) == (50, False, False) + + first = _row("a/main.py", "a") + again = _row("a/main.py", "b", found_by_ai=True) + chosen, left_out = choose_rows([_row("z/main.py", "z"), first, again], limit=50) + assert [row.candidate.entry for row in chosen] == ["a/main.py", "z/main.py"] + assert chosen[0] is first + assert left_out is True + chosen, left_out = choose_rows(rows, limit=3) + assert (len(chosen), left_out) == (3, True) + assert choose_rows(rows[:3]) == (sorted(rows[:3], key=lambda row: row.candidate.entry), False) + + +def test_the_models_rows_are_chosen_first_only_when_asked() -> None: + graph = _row("a/graph.py", "a") + helpers = [_row(f"{name}/main.py", name, sign="client", builds=()) for name in ("b", "c")] + found = _row("z/agent.ts", "z", found_by_ai=True, builds=()) + in_tests = _row("tests/agent.ts", "tests", side=True, found_by_ai=True, builds=()) + rows = [graph, *helpers, found, in_tests] + + def entries(chosen: list[Row]) -> list[str]: + return [row.candidate.entry for row in chosen] + + plain, plain_left_out = choose_rows(rows, limit=3) + first, first_left_out = choose_rows(rows, limit=3, models_first=True) + many = [_row(f"h{i:02}/main.py", f"h{i:02}", sign="client", builds=()) for i in range(49)] + body, dropped = finish( + rows=[graph, *many, found], + root_name="shop", + commit_sha=SHA, + truncated=False, + error=None, + models_first=True, + ) + written = [agent["entry"] for agent in json.loads(body)["agents"]] + + assert _confidence(graph, *helpers, found) == ["high", "medium", "medium", "low"] + assert (entries(plain), plain_left_out) == (["a/graph.py", "b/main.py", "c/main.py"], True) + assert (entries(first), first_left_out) == (["a/graph.py", "z/agent.ts", "b/main.py"], True) + # Which rows are kept changes; the order they are written in does not. + assert dropped is True + assert written == ["a/graph.py", *(f"h{i:02}/main.py" for i in range(48)), "z/agent.ts"] + + +def test_a_row_the_platform_rules_reject_is_dropped_with_truncated(caplog: pytest.LogCaptureFixture) -> None: + good = _row("good/main.py", "good") + bad = [ + _row("bad/../escape.py", "bad"), + _row("label/main.py", "label", framework={"name": "langgraph", "version": None}), + _row("env/main.py", "env", env_names=("BAD-NAME-sk-fake-123",)), + _row("tools/main.py", "tools", tools=-1), + _row("word/main.py", "word", framework=_framework("autogen")), + ] + + with caplog.at_level(logging.WARNING, logger="fi.alk.harness.scan_document"): + document, _, dropped = _finish([good, *bad]) + + assert [agent["entry"] for agent in document["agents"]] == ["good/main.py"] + assert document["status"] == "found" + assert document["truncated"] is True + assert dropped is True + left_out = [record.getMessage() for record in caplog.records] + assert len(left_out) == len(bad) + assert all(message.startswith("agent scan: row left out, problem=") for message in left_out) + assert "sk-fake-123" not in caplog.text + assert "escape" not in caplog.text + + document, _, dropped = _finish(bad) + assert (document["status"], document["agents"], document["truncated"]) == ("none", [], True) + assert dropped is True + + +def test_a_bad_top_level_raises(monkeypatch: pytest.MonkeyPatch) -> None: + def problem(**changes: Any) -> str: + arguments: dict[str, Any] = { + "rows": [], + "root_name": "shop", + "commit_sha": SHA, + "truncated": False, + "error": None, + } + arguments.update(changes) + with pytest.raises(ScanResultInvalid) as raised: + finish(**arguments) + assert raised.value.problem in str(raised.value) + return raised.value.problem + + assert problem(error=("Scan-Failed", "The scan failed.")) == "error.code.pattern" + assert problem(error=("scan_failed", " ")) == "error.message.blank" + assert problem(error=("scan_failed", "m" * 2001)) == "error.message.too_long" + assert problem(error=("scan_failed", "sk-fake-123\x00")) == "error.message.null_byte" + assert "sk-fake-123" not in problem(error=("sk-fake-123", "The scan failed.")) + + monkeypatch.setattr(sd, "MAX_BYTES", 100) + assert problem(rows=[_row("main.py")]) == "document.too_large" + + assert wire_problems([]) == ["not_an_object"] + assert wire_problems({"schema_version": "futureagi.agent-scan.v1"}) == [ + "status.missing", + "commit_sha.missing", + "agents.missing", + "truncated.missing", + "error.missing", + ] + + +def test_the_file_matches_the_platform_serializer() -> None: + fixture = json.loads(FIXTURE.read_text(encoding="utf-8")) + assert fixture["serializer_commit"] == "e07d56fcd" + cases = fixture["cases"] + assert len(cases) > 100 + assert {case["accepted"] for case in cases} == {True, False} + # Lone surrogates and null bytes reach the reader only as escapes in the text. + assert any("\\ud800" in case["document_text"] for case in cases) + assert any("\\u0000" in case["document_text"] for case in cases) + + disagreements = [] + for case in cases: + problems = wire_problems(json.loads(case["document_text"])) + if (problems == []) != case["accepted"]: + disagreements.append((case["name"], case["accepted"], problems)) + assert all(PROBLEM_NAME.fullmatch(problem) for problem in problems), case["name"] + assert disagreements == [] + + +def test_a_failed_write_leaves_no_agents_json( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _, body, _ = _finish([_row("main.py")]) + + written = write_agents_json(tmp_path / "ok", body) + assert written == tmp_path / "ok" / "agents.json" + assert written.read_bytes() == body + assert stat.S_IMODE(written.stat().st_mode) == 0o644 + assert [path.name for path in (tmp_path / "ok").iterdir()] == ["agents.json"] + + def refuse(*args: Any, **kwargs: Any) -> None: + raise OSError("disk full") + + for name in ("replace", "fsync"): + output = tmp_path / name + output.mkdir() + with monkeypatch.context() as patch: + patch.setattr(sd.os, name, refuse) + with pytest.raises(OSError): + write_agents_json(output, body) + assert list(output.iterdir()) == [] + + earlier = tmp_path / "earlier" + earlier.mkdir() + (earlier / "agents.json").write_bytes(b"{}") + with monkeypatch.context() as patch: + patch.setattr(sd.os, "replace", refuse) + with pytest.raises(OSError): + write_agents_json(earlier, body) + assert [path.name for path in earlier.iterdir()] == ["agents.json"] + assert (earlier / "agents.json").read_bytes() == b"{}" + + +def test_a_disambiguating_suffix_survives_the_255_cut() -> None: + document, _, _ = _finish([_row("a/main.py"), _row("b/main.py")], root="x" * 300) + + names = sorted(agent["name"] for agent in document["agents"]) + assert names == [ + "x" * (255 - len(" (a/main.py)")) + " (a/main.py)", + "x" * (255 - len(" (b/main.py)")) + " (b/main.py)", + ] + + +def test_a_chat_model_row_is_low_with_its_own_sentence() -> None: + evidence = (("model langchain", "app/bot.py", 4),) + + agent = _finish( + [ + _row( + "app/bot.py", + "app", + sign="model", + builds=(), + evidence=evidence, + framework=_framework("langchain"), + ) + ] + )[0]["agents"][0] + + assert agent["confidence"] == "low" + assert agent["evidence"] == ["Calls a chat model through LangChain in app/bot.py, line 4."] + assert sd._sentence("model no such!", "app.py", 1) == ( + "Calls a chat model through an agent framework in app.py, line 1." + ) diff --git a/tests/harness/test_scan_folders.py b/tests/harness/test_scan_folders.py new file mode 100644 index 000000000..079dd6666 --- /dev/null +++ b/tests/harness/test_scan_folders.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +import multiprocessing.process +import os +import signal +import time +from pathlib import Path +from typing import Any + +import pytest + +from fi.alk.harness import scan_folders +from fi.alk.harness.framework_detection import detect_framework +from fi.alk.harness.scan_folders import ( + ClockRanOut, + FolderChecker, + FolderCheckerBroken, + FolderWord, +) +from fi.alk.harness.source_folder import agent_folder + +CREW = "from crewai import Crew\nCrew()\n" +AUTOGEN = "from autogen_agentchat.agents import AssistantAgent\nAssistantAgent()\n" +OPENAI_AGENT = "from agents import Agent\nAgent()\n" +PLAIN = "print('hello')\n" +SITE = ".venv/lib/python3.12/site-packages" +# The first check in a session also starts the fork server, which imports the kit. +FIRST_CHECK_SECONDS = 120.0 +# Killing and reaping a stopped child, on a loaded machine. +KILL_MARGIN_SECONDS = 4.0 +UNKNOWN = {"name": "unknown", "version": None, "label": None} +FOLDERS = { + "": None, + "crew": {"name": "crewai", "version": None, "label": None}, + "autogen": {"name": "other", "version": None, "label": "autogen"}, + "oa": {"name": "openai_agents", "version": None, "label": None}, + "plain": {"name": "none", "version": None, "label": None}, +} + + +def _write(folder: Path, files: dict[str, str]) -> None: + for relative, content in files.items(): + path = folder / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def _repo(tmp_path: Path) -> Path: + source = tmp_path / "repo" + _write( + source, + { + "crew/main.py": CREW, + "autogen/main.py": AUTOGEN, + "oa/main.py": OPENAI_AGENT, + "plain/main.py": PLAIN, + # Installed packages without openai-agents: a check that read them would drop + # the `agents` import, so the word shows which package folder the child used. + f"{SITE}/requests-2.0.dist-info/METADATA": "Version: 2.0\n", + f"oa/{SITE}/requests-2.0.dist-info/METADATA": "Version: 2.0\n", + }, + ) + return source + + +def _later(seconds: float) -> float: + return time.monotonic() + seconds + + +def _gone(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + return False + + +def _kill_if_alive(pid: int | None) -> None: + if pid is not None and not _gone(pid): + os.kill(pid, signal.SIGKILL) + + +def _exits_three(conn: Any) -> None: + os._exit(3) + + +def _refuses_to_start(self: Any) -> None: + raise OSError("no processes left") + + +def test_each_folder_gets_the_builds_framework_word(tmp_path: Path) -> None: + source = _repo(tmp_path) + words: dict[str, FolderWord] = {} + pids = set() + with FolderChecker() as checker: + for subfolder in FOLDERS: + words[subfolder] = checker.check( + source, subfolder, deadline=_later(FIRST_CHECK_SECONDS) + ) + pids.add(checker.pid) + + for subfolder, expected in FOLDERS.items(): + answer = detect_framework( + agent_folder(source, subfolder) if subfolder else source, + repo_root=source, + package_dir=None, + ) + assert words[subfolder] == FolderWord( + {"name": answer["name"], "version": None, "label": answer.get("label")}, + finished=True, + ) + assert set(words[subfolder].framework) == {"name", "version", "label"} + if expected is not None: + assert words[subfolder].framework == expected + assert len(pids) == 1 + assert None not in pids and os.getpid() not in pids + + +def test_the_clock_ends_a_stopped_child(tmp_path: Path) -> None: + source = _repo(tmp_path) + stopped: int | None = None + try: + with FolderChecker() as checker: + checker.check(source, "crew", deadline=_later(FIRST_CHECK_SECONDS)) + stopped = checker.pid + assert stopped is not None + os.kill(stopped, signal.SIGSTOP) + + started = time.monotonic() + with pytest.raises(ClockRanOut): + checker.check(source, "plain", deadline=started + 1.0) + + assert time.monotonic() - started < 1.0 + KILL_MARGIN_SECONDS + assert checker.pid is None + assert _gone(stopped) + + with pytest.raises(ClockRanOut): + checker.check(source, "autogen", deadline=time.monotonic() - 1.0) + assert checker.pid is None + + word = checker.check(source, "plain", deadline=_later(FIRST_CHECK_SECONDS)) + assert word == FolderWord(FOLDERS["plain"], finished=True) + assert checker.pid not in (None, stopped) + finally: + _kill_if_alive(stopped) + + +def test_the_child_is_restarted_after_a_kill(tmp_path: Path) -> None: + source = _repo(tmp_path) + killed: int | None = None + try: + with FolderChecker() as checker: + first = checker.check(source, "crew", deadline=_later(FIRST_CHECK_SECONDS)) + assert first == FolderWord(FOLDERS["crew"], finished=True) + killed = checker.pid + assert killed is not None + os.kill(killed, signal.SIGKILL) + + word = checker.check(source, "autogen", deadline=_later(FIRST_CHECK_SECONDS)) + + assert word == FolderWord(UNKNOWN, finished=False) + assert _gone(killed) + + again = checker.check(source, "plain", deadline=_later(FIRST_CHECK_SECONDS)) + assert again == FolderWord(FOLDERS["plain"], finished=True) + assert checker.pid not in (None, killed) + finally: + _kill_if_alive(killed) + + +def test_a_child_that_cannot_start_is_broken( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = _repo(tmp_path) + + # The target is looked up when the child starts, so the child runs this one and exits 3. + monkeypatch.setattr(scan_folders, "_serve", _exits_three) + with FolderChecker() as checker: + with pytest.raises(FolderCheckerBroken, match="exit code 3"): + checker.check(source, "crew", deadline=_later(FIRST_CHECK_SECONDS)) + assert checker.pid is None + monkeypatch.undo() + + monkeypatch.setattr(multiprocessing.process.BaseProcess, "start", _refuses_to_start) + with FolderChecker() as checker: + with pytest.raises(FolderCheckerBroken, match="could not start"): + checker.check(source, "crew", deadline=_later(FIRST_CHECK_SECONDS)) + assert checker.pid is None + + +def test_exit_never_raises(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + source = _repo(tmp_path) + + assert FolderChecker().__exit__(None, None, None) is None + + with FolderChecker() as checker: + checker.check(source, "crew", deadline=_later(FIRST_CHECK_SECONDS)) + ended = checker.pid + assert ended is not None + assert checker.pid is None + assert _gone(ended) + assert checker.__exit__(None, None, None) is None + + stuck = FolderChecker() + stuck.check(source, "crew", deadline=_later(FIRST_CHECK_SECONDS)) + alive = stuck.pid + assert alive is not None + real_kill = os.kill + + def refuse(pid: int, sig: int) -> None: + raise PermissionError("not allowed") + + monkeypatch.setattr(os, "kill", refuse) + try: + assert stuck.__exit__(None, None, None) is None + assert stuck.pid is None + finally: + monkeypatch.undo() + try: + real_kill(alive, signal.SIGKILL) + except ProcessLookupError: + pass + + +def test_a_fault_in_the_check_comes_back_as_a_trace(tmp_path, monkeypatch) -> None: + """Only a folder the build refuses is a plain ``unknown``; a fault inside the framework + check itself (here in ``folder_findings``, below detect_framework's own catch) is sent + back with its trace, so the parent logs it and marks the row unfinished.""" + import fi.alk.harness.framework_detection as fd + + def broken(*args, **kwargs): + raise KeyError("a fault in the check") + + monkeypatch.setattr(fd, "folder_findings", broken) + (tmp_path / "agent.py").write_text(CREW, encoding="utf-8") + assert fd.detect_framework(tmp_path) == {"name": "unknown", "version": None} + answer = scan_folders._answer(tmp_path, "") + assert set(answer) == {scan_folders._FAILED} + assert "KeyError" in answer[scan_folders._FAILED] + + assert scan_folders._answer(tmp_path, "no/such/folder") == {"name": "unknown", "version": None} + + +def test_a_failed_answer_marks_the_row_unfinished_and_logs_the_trace(tmp_path, caplog) -> None: + """The parent's half: a child answer carrying a trace becomes an unfinished ``unknown``, + kept for the folder, with the trace in the log.""" + + class _Conn: + def send(self, message: object) -> None: + pass + + def poll(self, timeout: float) -> bool: + return True + + def recv(self) -> object: + return {scan_folders._FAILED: "Traceback (most recent call last):\nKeyError: 'x'"} + + checker = FolderChecker() + checker._process = object() # a started child, as far as check() can tell + checker._conn = _Conn() + try: + with caplog.at_level("WARNING", logger=scan_folders.logger.name): + word = checker.check(tmp_path, "", deadline=time.monotonic() + 30) + finally: + checker._process = None + checker._conn = None + + assert word.finished is False + assert word.framework["name"] == "unknown" + assert "the check failed in the child" in caplog.text and "KeyError" in caplog.text + assert checker.check(tmp_path, "", deadline=time.monotonic() + 30).finished is False diff --git a/tests/harness/test_scan_walk.py b/tests/harness/test_scan_walk.py new file mode 100644 index 000000000..6231c446f --- /dev/null +++ b/tests/harness/test_scan_walk.py @@ -0,0 +1,1393 @@ +from __future__ import annotations + +import ast +import inspect +import io +import json +import os +import tokenize +from pathlib import Path + +import pytest + +from fi.alk.harness import framework_detection as fd +from fi.alk.harness import scan_walk +from fi.alk.harness import source_folder +from fi.alk.harness.scan_walk import ( + Candidate, + WalkResult, + build_hazard, + describe_entry, + fits_path_rule, + walk_repo, +) + +GRAPH = "from langgraph.graph import StateGraph\nStateGraph(dict)\n" +CLIENT = "from openai import OpenAI\nOpenAI()\n" +FOREVER = float("inf") +ALLOWANCE = 1_000_000_000 + + +def _write(folder: Path, files: dict[str, str]) -> None: + for relative, content in files.items(): + path = folder / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def _walk(source: Path, **kwargs: object) -> WalkResult: + return walk_repo(source, deadline=FOREVER, **kwargs) # type: ignore[arg-type] + + +def _entries(result: WalkResult) -> list[str]: + return [agent.entry for agent in result.agents] + + +def _agent(result: WalkResult, entry: str) -> Candidate: + return next(agent for agent in result.agents if agent.entry == entry) + + +def _snapshot(folder: Path) -> list[tuple[str, int, int]]: + seen = [] + for directory, directories, files in os.walk(folder, followlinks=False): + for name in sorted(directories + files): + path = os.path.join(directory, name) + status = os.lstat(path) + seen.append((path, status.st_size, status.st_mtime_ns)) + return sorted(seen) + + +def _token_work(source: str) -> int: + tokens = tokenize.tokenize(io.BytesIO(source.encode()).readline) + return sum(len(token.line) for token in tokens) + + +def test_the_walk_skips_what_the_build_skips_and_never_follows_a_link( + tmp_path: Path, +) -> None: + outside = tmp_path / "outside" + _write(outside, {"agent_graph.py": GRAPH}) + source = tmp_path / "source" + ran = tmp_path / "ran" + _write( + source, + { + "app/graph.py": GRAPH + f"open({str(ran)!r}, 'w').close()\n", + "app/build/graph.py": GRAPH, + "build/graph.py": GRAPH, + "env/graph.py": GRAPH, + "node_modules/pkg/graph.py": GRAPH, + ".venv/lib/graph.py": GRAPH, + "lib/site-packages/graph.py": GRAPH, + "lib/site/pkg-1.0.dist-info/METADATA": "Name: pkg\n", + "lib/site/graph.py": GRAPH, + }, + ) + (source / "linked").symlink_to(outside, target_is_directory=True) + (source / "linked_graph.py").symlink_to(outside / "agent_graph.py") + before = _snapshot(tmp_path) + + result = _walk(source) + + assert sorted(_entries(result)) == ["app/build/graph.py", "app/graph.py"] + assert result.unread_files == 1 + assert result.stopped == "" + assert not ran.exists() + assert _snapshot(tmp_path) == before + + +def test_a_heavy_file_uses_only_its_own_allowance(tmp_path: Path) -> None: + # Every mention of `C`, in its imports and its uses, walks all of its targets. + targets = 500 + imports = "".join(f"import crewai.m{index} as C\n" for index in range(targets)) + light = imports + "C.Crew()\n" * 100 + heavy = imports + "C.Crew()\n" * 1_500 + per_mention = fd._WORK_PER_TARGET * targets + light_work = (targets + 100) * per_mention + 3 * _token_work(light) + heavy_work = (targets + 1_500) * per_mention + assert scan_walk._FILE_WORK == ALLOWANCE + assert light_work < ALLOWANCE < 2 * light_work + assert ALLOWANCE < heavy_work < fd._MAX_WORK + _write(tmp_path, {"a_light.py": light, "b_light.py": light, "c_heavy.py": heavy}) + + result = _walk(tmp_path) + + assert _entries(result) == ["a_light.py", "b_light.py"] + assert result.unread_files == 1 + assert result.stopped == "" + + +def test_the_private_detector_names_it_uses_exist() -> None: + for name in ( + "_ALWAYS_SKIPPED", + "_TOP_ONLY_SKIPS", + "_FileScan", + "_has_long_line", + "_is_side_file", + "_is_side_part", + "_OverBudget", + "_own_modules", + "_read_file", + "_scan_file", + "_visible_tokens", + ): + assert hasattr(fd, name), name + assert hasattr(source_folder, "_FOLDER_FORM") + assert source_folder._FOLDER_MAX == 512 + assert list(inspect.signature(fd._FileScan).parameters) == [ + "shadowed", + "outside_names", + "work", + ] + assert "read_uses" in inspect.signature(fd._scan_file).parameters + + +@pytest.mark.parametrize( + "shape", + [ + b'x = """never closed\n', + b"if True:\n a = 1\n b = 2\n", + b"if True:\n\ta = 1\n b = 2\n", + b"# caf\xe9\nx = 1\n", + b"x = 1\x00\n", + ], + ids=["unclosed_string", "bad_dedent", "tab_and_spaces", "latin1_no_coding", "null"], +) +def test_a_file_the_tokenizer_rejects_is_unread_not_a_crash( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, shape: bytes +) -> None: + (tmp_path / "a_shape.py").write_bytes(GRAPH.encode() + shape) + _write(tmp_path, {"z_agent.py": GRAPH}) + + result = _walk(tmp_path) + + assert result.stopped == "" + assert "z_agent.py" in _entries(result) + assert ("a_shape.py" in _entries(result)) != (result.unread_files == 1) + + def rejects(data: bytes, work: object) -> object: + raise LookupError("a reader fault of any kind") + + monkeypatch.setattr(scan_walk, "_visible_tokens", rejects) + failing = _walk(tmp_path) + + assert failing.stopped == "" + assert failing.agents == () + assert failing.unread_files == 2 + + +def test_the_token_pass_writes_no_warnings( + tmp_path: Path, capfd: pytest.CaptureFixture[str], recwarn: pytest.WarningsRecorder +) -> None: + _write( + tmp_path, + {"agent.py": CLIENT + ("import os\nos.getenv('A\\{')\nx = f'\\{a}'\n" * 3)}, + ) + + _walk(tmp_path) + + assert not recwarn.list + assert "Warning" not in capfd.readouterr().err + + +def test_a_relative_import_joins_the_sub_agent(tmp_path: Path) -> None: + _write( + tmp_path, + { + "app/__init__.py": "", + "app/main.py": GRAPH + + "from . import planner\n" + + "def build():\n from .researcher import agent\n return agent\n", + "app/planner.py": GRAPH, + "app/researcher.py": GRAPH, + "deep/a/b/main.py": GRAPH + + "from ..helpers import util\nfrom .. import (shared)\n", + "deep/a/helpers/util.py": GRAPH, + "deep/a/shared.py": GRAPH, + "pkg/pyproject.toml": "", + "pkg/run.py": GRAPH + "import tools.search\n", + "pkg/src/tools/search.py": GRAPH, + }, + ) + + result = _walk(tmp_path) + + assert sorted(_entries(result)) == ["app/main.py", "deep/a/b/main.py", "pkg/run.py"] + + +def test_settings_tools_and_clients_are_read(tmp_path: Path) -> None: + source = tmp_path / "one" + _write( + source, + { + "bot.py": "import os\n" + + CLIENT + + "a = os.getenv('FIRST')\n" + + 'b = os.environ.get("SECOND")\n' + + "c = os.environ['THIRD']\n" + + "d = os.getenv(f'FOURTH')\n" + + "e = os.getenv('FIF' 'TH')\n" + + "f = os.getenv('SIX' + 'TH')\n" + + "g = os.getenv(NAME)\n" + + "h = os.environ[b'EIGHTH']\n" + + "def handler():\n return os.getenv( 'NINTH' )\n" + + "@tool\ndef one(): ...\n" + + "@function_tool\ndef two(): ...\n" + + "@mcp.tool()\ndef three(): ...\n" + + "@agents.function_tool(name='x')\ndef four(): ...\n" + + "@toolkit\ndef not_a_tool(): ...\n" + }, + ) + + [agent] = _walk(source).agents + + assert agent.sign == "client" + assert agent.env_names == ("FIRST", "NINTH", "SECOND", "THIRD") + assert agent.tools == 4 + assert [(kind, path) for kind, path, _ in agent.evidence] == [ + ("client openai", "bot.py") + ] + + forms = { + "openai.py": "OpenAI()", + "openai_async.py": "AsyncOpenAI()", + "openai_azure.py": "openai.AzureOpenAI()", + "anthropic.py": "Anthropic()", + "anthropic_async.py": "AsyncAnthropic()", + "genai.py": "genai.Client()", + "mistral.py": "Mistral(api_key=k)", + "cohere.py": "cohere.Client()", + "cohere_v2.py": "cohere.ClientV2()", + "groq.py": "Groq()", + "litellm.py": "litellm.completion(model=m)", + "litellm_async.py": "litellm.acompletion(model=m)", + } + clients = tmp_path / "clients" + _write(clients, {name: call + "\n" for name, call in forms.items()}) + _write(clients, {"not_a_client.py": "other.Client()\nOpenAI\n"}) + + assert sorted(_entries(_walk(clients))) == sorted(forms) + + +def test_a_getenv_default_is_never_an_env_name(tmp_path: Path) -> None: + _write( + tmp_path, + { + "bot.py": "import os\n" + + CLIENT + + "key = os.getenv('API_KEY', 'sk-fake-123')\n" + + "url = os.getenv('BASE_URL', 'FALLBACK_NAME')\n" + + "region = os.environ.get('REGION', 'DEFAULT_REGION')\n" + + "model = os.getenv('MODEL', default='DEFAULT_MODEL')\n" + }, + ) + + [agent] = _walk(tmp_path).agents + + assert agent.env_names == ("API_KEY", "BASE_URL", "MODEL", "REGION") + assert "sk-fake-123" not in repr(agent) + + +def test_env_files_with_values_are_never_opened( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write( + tmp_path, + { + ".env": "REAL_SECRET=sk-fake-123\n", + ".env.local": "LOCAL_SECRET=sk-fake-456\n", + "app/.env": "APP_SECRET=sk-fake-789\n", + ".env.example": "# a note\nEXAMPLE_NAME=sk-fake-000\n" + "export EXPORTED_NAME=1\n BAD-NAME=2\n", + "bot.py": CLIENT, + }, + ) + opened: list[str] = [] + read = scan_walk._read_file + + def spy(path: str, limit: int) -> bytes: + opened.append(os.path.basename(path)) + return read(path, limit) + + monkeypatch.setattr(scan_walk, "_read_file", spy) + + result = _walk(tmp_path) + + assert ".env.example" in opened + assert ".env" not in opened + assert ".env.local" not in opened + [agent] = result.agents + assert agent.env_names == ("EXAMPLE_NAME", "EXPORTED_NAME") + assert "sk-fake" not in repr(result) + + +@pytest.mark.parametrize( + "content", + [b"{not json", b"[" * 100_000 + b"]" * 100_000, b'{"graphs": "\xff"}'], + ids=["not_json", "deep", "not_utf8"], +) +def test_a_broken_langgraph_json_is_unread_not_a_crash( + tmp_path: Path, content: bytes +) -> None: + (tmp_path / "langgraph.json").write_bytes(content) + _write(tmp_path, {"graph.py": GRAPH}) + + result = _walk(tmp_path) + + assert result.stopped == "" + assert result.unread_files == 1 + assert _entries(result) == ["graph.py"] + + +def test_the_walk_limit_and_the_clock_stop_it( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + assert (scan_walk._MAX_FILES, scan_walk._MAX_BYTES) == (40_000, 268_435_456) + _write(tmp_path, {"a.py": GRAPH, "b.py": GRAPH, "c.py": GRAPH}) + calls: list[int] = [] + + def clock() -> float: + calls.append(1) + return 0.0 if len(calls) == 1 else 100.0 + + timed = walk_repo(tmp_path, deadline=10.0, clock=clock) + + assert timed.stopped == "clock" + assert _entries(timed) == ["a.py"] + # The walk checked before each of its two files; the join after it checks the clock too. + assert len(calls) >= 2 + + monkeypatch.setattr(scan_walk, "_MAX_FILES", 2) + by_count = _walk(tmp_path) + assert by_count.stopped == "limit" + assert _entries(by_count) == ["a.py", "b.py"] + + monkeypatch.setattr(scan_walk, "_MAX_FILES", 40_000) + monkeypatch.setattr(scan_walk, "_MAX_BYTES", len(GRAPH)) + by_bytes = _walk(tmp_path) + assert by_bytes.stopped == "limit" + assert _entries(by_bytes) == ["a.py"] + + +def test_a_memory_error_stops_the_walk_with_its_facts_dropped( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write(tmp_path, {"a.py": GRAPH, "b.py": GRAPH}) + walks: list[scan_walk._Walk] = [] + + class Recording(scan_walk._Walk): + def __init__(self, *args: object, **kwargs: object) -> None: + super().__init__(*args, **kwargs) # type: ignore[arg-type] + walks.append(self) + + read = scan_walk._read_file + + def runs_out(path: str, limit: int) -> bytes: + if path.endswith("b.py"): + raise MemoryError() + return read(path, limit) + + monkeypatch.setattr(scan_walk, "_Walk", Recording) + monkeypatch.setattr(scan_walk, "_read_file", runs_out) + + result = _walk(tmp_path) + + assert result.stopped == "memory" + assert result.agents == () + [walk] = walks + assert walk.facts == {} + + +def test_a_weak_candidate_does_not_swallow_a_strong_one(tmp_path: Path) -> None: + _write( + tmp_path, + { + "app/pyproject.toml": "", + "app/graph.py": GRAPH + "from llm import ask\n", + "app/llm.py": CLIENT, + "evaluation/pyproject.toml": "", + "evaluation/run.py": CLIENT + "from app.graph import graph\n", + }, + ) + + result = _walk(tmp_path) + + assert sorted(_entries(result)) == ["app/graph.py", "evaluation/run.py"] + + +def test_a_side_folder_script_does_not_swallow_the_agent(tmp_path: Path) -> None: + _write( + tmp_path, + { + "app/graph.py": GRAPH, + "evals/run.py": GRAPH + "from app.graph import graph\n", + }, + ) + + result = _walk(tmp_path) + + assert [(agent.entry, agent.side) for agent in result.agents] == [ + ("app/graph.py", False), + ("evals/run.py", True), + ] + assert result.candidate_outside_side + + +def test_an_import_cycle_is_one_agent(tmp_path: Path) -> None: + direct = tmp_path / "direct" + _write( + direct, + {"long_name.py": GRAPH + "import b\n", "b.py": GRAPH + "import long_name\n"}, + ) + through = tmp_path / "through" + _write( + through, + { + "c2.py": GRAPH + "import c1\n", + "c1.py": GRAPH + "import helper\n", + "helper.py": "import c2\n", + }, + ) + + assert _entries(_walk(direct)) == ["b.py"] + assert _entries(_walk(through)) == ["c1.py"] + + +def test_a_client_beside_a_stronger_agent_is_not_listed(tmp_path: Path) -> None: + _write( + tmp_path, + { + "pyproject.toml": "", + "app/graph.py": GRAPH, + "tools/backfill.py": CLIENT, + "other/pyproject.toml": "", + "other/bot.py": CLIENT, + }, + ) + + result = _walk(tmp_path) + + assert sorted(_entries(result)) == ["app/graph.py", "other/bot.py"] + + +def test_a_root_test_file_is_side(tmp_path: Path) -> None: + _write( + tmp_path, + {"test_agent.py": GRAPH, "agent_test.py": GRAPH, "conftest.py": GRAPH}, + ) + + result = _walk(tmp_path) + + assert {agent.entry: (agent.side, agent.subfolder) for agent in result.agents} == { + "agent_test.py": (True, ""), + "conftest.py": (True, ""), + "test_agent.py": (True, ""), + } + assert not result.candidate_outside_side + assert not result.source_outside_side + + +def test_the_project_folder_rule(tmp_path: Path) -> None: + one = tmp_path / "one" + _write( + one, + {"pyproject.toml": "", "app/graph.py": GRAPH, "scripts/run.py": GRAPH}, + ) + two = tmp_path / "two" + _write( + two, + { + "svc/pyproject.toml": "", + "svc/a/x.py": GRAPH, + "svc/b/y.py": GRAPH, + "lone/requirements.txt": "", + "lone/deep/z.py": GRAPH, + "plain/graph.py": GRAPH, + }, + ) + + first = _walk(one) + second = _walk(two) + + assert {agent.entry: agent.subfolder for agent in first.agents} == { + "app/graph.py": "", + "scripts/run.py": "scripts", + } + assert ("project pyproject.toml", "pyproject.toml", 0) in _agent( + first, "app/graph.py" + ).evidence + assert {agent.entry: agent.subfolder for agent in second.agents} == { + "svc/a/x.py": "svc/a", + "svc/b/y.py": "svc/b", + "lone/deep/z.py": "lone", + "plain/graph.py": "plain", + } + + +def test_two_graphs_in_one_langgraph_json_mark_the_folder(tmp_path: Path) -> None: + _write( + tmp_path, + { + "svc/pyproject.toml": "", + "svc/langgraph.json": json.dumps( + {"graphs": {"a": "./a.py:graph", "b": "./b.py:graph"}} + ), + "svc/a.py": "graph = None\n", + "svc/b.py": "graph = None\n", + "one/langgraph.json": json.dumps({"graphs": {"a": "./a.py:graph"}}), + "one/a.py": "graph = None\n", + }, + ) + + result = _walk(tmp_path) + + assert [ + (agent.entry, agent.sign, agent.subfolder, agent.build_may_stop) + for agent in result.agents + ] == [ + ("one/a.py", "graph", "one", ""), + ("svc/a.py", "graph", "svc", "graphs"), + ("svc/b.py", "graph", "svc", "graphs"), + ] + assert ("graph", "svc/langgraph.json", 0) in _agent(result, "svc/a.py").evidence + assert build_hazard(tmp_path, "svc") == "graphs" + assert build_hazard(tmp_path, "one") == "" + + +def test_two_agent_py_files_mark_the_folder(tmp_path: Path) -> None: + _write( + tmp_path, + { + "bot/requirements.txt": "", + "bot/agent.py": CLIENT, + "bot/tests/agent.py": "x = 1\n", + "single/agent.py": CLIENT, + "single/node_modules/pkg/agent.py": "", + "single/.venv/agent.py": "", + "single/__pycache__/agent.py": "", + "kept/agent.py": "", + "kept/venv/agent.py": "", + }, + ) + + result = _walk(tmp_path) + + assert { + agent.entry: (agent.sign, agent.subfolder, agent.build_may_stop) + for agent in result.agents + } == { + "bot/agent.py": ("agent_py", "bot", "agent_py"), + "single/agent.py": ("agent_py", "single", ""), + } + assert build_hazard(tmp_path, "kept") == "agent_py" + + +def test_more_than_fifty_agents_in_one_folder_keeps_fifty(tmp_path: Path) -> None: + _write(tmp_path, {f"bot_{index:02d}.py": CLIENT for index in range(51)}) + + over = _walk(tmp_path) + + assert _entries(over) == [f"bot_{index:02d}.py" for index in range(50)] + assert over.more_than_fifty + + (tmp_path / "bot_50.py").unlink() + exact = _walk(tmp_path) + + assert len(exact.agents) == 50 + assert not exact.more_than_fifty + + +def test_paths_the_path_rule_rejects_are_left_out(tmp_path: Path) -> None: + long_folder = "/".join(["d" * 100] * 5) + _write( + tmp_path, + { + "main.py": GRAPH, + "ok/graph.py": GRAPH, + "my agent/graph.py": GRAPH, + "svc\n/graph.py": GRAPH, + f"{long_folder}/graph.py": GRAPH, + }, + ) + + result = _walk(tmp_path) + + assert {agent.entry: agent.subfolder for agent in result.agents} == { + "main.py": "", + "ok/graph.py": "ok", + } + assert result.paths_left_out == 3 + assert fits_path_rule("ok/graph.py") + assert fits_path_rule("x" * 512) + assert not fits_path_rule("") + assert not fits_path_rule("svc\n") + assert not fits_path_rule("a/../b") + assert not fits_path_rule("x" * 513) + + +def test_an_upload_keeps_one_agent_at_the_root_counted_for_the_root( + tmp_path: Path, +) -> None: + source = tmp_path / "repo" + _write( + source, + { + "svc/pyproject.toml": "", + "svc/graph.py": GRAPH + "import os\nos.getenv('SVC_KEY')\n", + "main.py": "import os\nos.getenv('ROOT_KEY')\n@tool\ndef search(): ...\n", + "prompts/system.txt": "hello\n", + ".env.example": "TEMPLATE_KEY=x\n", + "tests/test_x.py": "import os\nos.getenv('TEST_KEY')\n", + "other/bot.py": CLIENT, + }, + ) + + uploaded = _walk(source, upload=True) + listed = _walk(source) + + [agent] = uploaded.agents + assert (agent.entry, agent.subfolder) == ("svc/graph.py", "") + assert agent.env_names == ("ROOT_KEY", "SVC_KEY", "TEMPLATE_KEY") + assert agent.tools == 1 + assert agent.prompt_files == 1 + assert agent.build_may_stop == build_hazard(source, "") + assert uploaded.upload_others == 1 + in_folder = _agent(listed, "svc/graph.py") + assert in_folder.subfolder == "svc" + assert in_folder.env_names == ("SVC_KEY", "TEMPLATE_KEY") + assert (in_folder.tools, in_folder.prompt_files) == (0, 0) + + side_only = tmp_path / "side" + _write( + side_only, + { + "scripts/run.py": CLIENT + "import os\nos.getenv('SIDE_KEY')\n", + "main.py": "import os\nos.getenv('MAIN_KEY')\n", + }, + ) + + [side] = _walk(side_only, upload=True).agents + + assert (side.entry, side.subfolder, side.side) == ("scripts/run.py", "", True) + assert side.env_names == ("MAIN_KEY", "SIDE_KEY") + + +def test_env_names_are_checked_and_ordered(tmp_path: Path) -> None: + names = [ + "Z_CODE", + "A_CODE", + "BAD NAME", + "ENDS\\n", + "1DIGIT", + "X" * 129, + "Y" * 128, + "A_CODE", + "naïve", + ] + code = "import os\n" + CLIENT + "".join(f"os.getenv('{n}')\n" for n in names) + _write( + tmp_path, + { + "svc/pyproject.toml": "", + "svc/bot.py": code, + "svc/.env.example": "B_TEMPLATE=1\nexport A_TEMPLATE=2\nA_CODE=3\n", + ".env.sample": "ROOT_TEMPLATE=1\n", + "elsewhere/.env.template": "ELSEWHERE=1\n", + }, + ) + + [agent] = _walk(tmp_path).agents + + assert agent.subfolder == "svc" + assert agent.env_names == ( + "A_CODE", + "Y" * 128, + "Z_CODE", + "A_TEMPLATE", + "B_TEMPLATE", + "ROOT_TEMPLATE", + ) + + +def test_describe_entry_reads_process_env(tmp_path: Path) -> None: + _write( + tmp_path, + { + "web/package.json": "{}\n", + "web/src/agent.ts": "const a = process.env.OPENAI_API_KEY;\n" + 'const b = process.env["OTHER_KEY"];\n' + "const c = process.env['THIRD_KEY'];\n" + "const d = process.env[name];\n" + "const e = myprocess.env.NOT_THIS;\n", + "web/src/util.mjs": "export const u = process.env.UTIL_KEY;\n", + "web/tests/agent.test.ts": "process.env.TEST_ONLY;\n", + "web/node_modules/pkg/index.js": "process.env.DEPENDENCY_KEY;\n", + "web/.env.example": "TEMPLATE_KEY=1\n", + "api/server.js": "process.env.ELSEWHERE;\n", + }, + ) + + candidate = describe_entry(tmp_path, "web/src/agent.ts", "web", deadline=FOREVER) + + assert candidate is not None + assert (candidate.sign, candidate.subfolder, candidate.side, candidate.partial) == ( + "client", + "web", + False, + False, + ) + assert candidate.env_names == ( + "OPENAI_API_KEY", + "OTHER_KEY", + "THIRD_KEY", + "UTIL_KEY", + "TEMPLATE_KEY", + ) + + +def test_describe_entry_keeps_the_models_folder_or_finds_package_json( + tmp_path: Path, +) -> None: + source = tmp_path / "repo" + _write( + source, + { + "pyproject.toml": "", + "web/package.json": "{}\n", + "web/src/agent.ts": "export {};\n", + "plain/deep/bot.py": CLIENT, + }, + ) + bare = tmp_path / "bare" + _write(bare, {"a/b/bot.py": CLIENT}) + + def folder_of(root: Path, entry: str, folder: str, upload: bool = False) -> str: + found = describe_entry(root, entry, folder, deadline=FOREVER, upload=upload) + assert found is not None + return found.subfolder + + assert folder_of(source, "web/src/agent.ts", "web/src") == "web/src" + assert folder_of(source, "web/src/agent.ts", "") == "" + assert folder_of(source, "web/src/agent.ts", "elsewhere") == "web" + assert folder_of(source, "web/src/agent.ts", "web/sr") == "web" + assert folder_of(source, "web/src/agent.ts", "web\n") == "web" + assert folder_of(source, "plain/deep/bot.py", "nowhere") == "" + assert folder_of(source, "web/src/agent.ts", "web/src", upload=True) == "" + assert folder_of(bare, "a/b/bot.py", "x") == "a/b" + assert describe_entry(source, "web/src/agent.ts\n", "web", deadline=FOREVER) is None + assert describe_entry(source, "", "", deadline=FOREVER) is None + + +def test_describe_entry_stops_at_its_deadline_with_partial( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + assert (scan_walk._DESCRIBE_FILES, scan_walk._DESCRIBE_BYTES) == (2_000, 33_554_432) + _write( + tmp_path, + { + "web/package.json": "{}\n", + "web/agent.ts": "process.env.FIRST;\n", + "web/b.ts": "process.env.SECOND;\n", + "web/c.ts": "process.env.THIRD;\n", + }, + ) + + late = describe_entry( + tmp_path, "web/agent.ts", "web", deadline=0.0, clock=lambda: 1.0 + ) + on_time = describe_entry( + tmp_path, "web/agent.ts", "web", deadline=10.0, clock=lambda: 1.0 + ) + monkeypatch.setattr(scan_walk, "_DESCRIBE_FILES", 1) + capped = describe_entry(tmp_path, "web/agent.ts", "web", deadline=FOREVER) + + assert late is not None and on_time is not None and capped is not None + assert (late.entry, late.partial, late.env_names) == ("web/agent.ts", True, ()) + assert not on_time.partial + assert on_time.env_names == ("FIRST", "SECOND", "THIRD") + assert (capped.partial, capped.env_names) == (True, ("FIRST",)) + + +# Every value line below carries "FAKE", so any name holding it is a piece of a value. +FAKE_KEY_LINE = "FAKEk3yAAAAAbbbbbbbbCCCCCCCCdddddddd0123456789EEEEEEEEffffffffGG" +AROUND = ("A_BEFORE", "Z_AFTER") +# Each shape: a settings template, and the names a reader of it should give, no more. +TEMPLATE_SHAPES: dict[str, tuple[str, tuple[str, ...]]] = { + "wrapped_bare_short_tail": ( + f"A_BEFORE=1\nSIGNING_KEY={FAKE_KEY_LINE}\n{FAKE_KEY_LINE}\nFAKEt4il=\nZ_AFTER=2\n", + (*AROUND, "SIGNING_KEY"), + ), + "wrapped_once_short_tail": ( + f"A_BEFORE=1\nSIGNING_KEY={FAKE_KEY_LINE}\nFAKEt4il=\nZ_AFTER=2\n", + (*AROUND, "SIGNING_KEY"), + ), + "bare_body_below_an_empty_setting": ( + f"A_BEFORE=1\nSIGNING_KEY=\n{FAKE_KEY_LINE}\nFAKEt4il=\nZ_AFTER=2\n", + (*AROUND, "SIGNING_KEY"), + ), + "crlf_wrapped_short_tail": ( + f"A_BEFORE=1\r\nKEY={FAKE_KEY_LINE}\r\nFAKEcb=\r\nZ_AFTER=2\r\n", + (*AROUND, "KEY"), + ), + "base64url_tail_with_underscore": ( + f"A_BEFORE=1\nJWK_D={FAKE_KEY_LINE}\nFAKE_url_tail_AAAAbbbbCCCCddddEEEEffff0123=\nZ_AFTER=2\n", + (*AROUND, "JWK_D"), + ), + "base64url_line_below_an_empty_setting": ( + "A_BEFORE=1\nJWK_D=\nFAKE_url_tail_AAAAbbbbCCCCddddEEEEffff0123=\nZ_AFTER=2\n", + (*AROUND, "JWK_D"), + ), + "base32_tail": ( + "A_BEFORE=1\nTOTP_SEED=FAKEBASE32AAAABBBBCCCCDDDDEEEE\nFAKEBASE32TAIL234567ABCDEFGH=\nZ_AFTER=2\n", + (*AROUND, "TOTP_SEED"), + ), + "triple_double_quotes": ( + f'A_BEFORE=1\nTRIPLE="""\n{FAKE_KEY_LINE}\nFAKEtq=\n"""\nZ_AFTER=2\n', + (*AROUND, "TRIPLE"), + ), + "triple_single_quotes": ( + f"A_BEFORE=1\nTRIPLE='''\n{FAKE_KEY_LINE}\nFAKEts=\n'''\nZ_AFTER=2\n", + (*AROUND, "TRIPLE"), + ), + "escaped_single_quote": ( + f"A_BEFORE=1\nESC='it\\'s\n{FAKE_KEY_LINE}\nFAKEesc=\n'\nZ_AFTER=2\n", + (*AROUND, "ESC"), + ), + "escaped_double_quote": ('A_BEFORE=1\nESC="say \\"hi\nFAKEesd=\n"\nZ_AFTER=2\n', (*AROUND, "ESC")), + "quoted_value_left_open": ('A_BEFORE=1\nKEY="\nFAKEdq=\n"\nZ_AFTER=2\n', (*AROUND, "KEY")), + "key_block_with_one_short_line": ( + "A_BEFORE=1\nPEM=-----BEGIN PRIVATE KEY-----\nFAKEpm=\n-----END PRIVATE KEY-----\nZ_AFTER=2\n", + (*AROUND, "PEM"), + ), + "padded_tail_below_a_short_line": ( + "A_BEFORE=1\nKEY=FAKEshortline\nFAKEpad==\nZ_AFTER=2\n", + (*AROUND, "KEY"), + ), + "key_block_opened_in_a_comment": ( + f"A_BEFORE=1\n#PEM=-----BEGIN PRIVATE KEY-----\n{FAKE_KEY_LINE}\nFAKEpc=\n" + "-----END PRIVATE KEY-----\nZ_AFTER=2\n", + AROUND, + ), + "line_joined_by_a_backslash": ( + f"A_BEFORE=1\nexport LONG={FAKE_KEY_LINE}\\\nFAKEbs=\nZ_AFTER=2\n", + (*AROUND, "LONG"), + ), + "lines_joined_by_backslashes": ( + "A_BEFORE=1\nexport OPTS=--first \\\n --second \\\n FAKEthird=x\nZ_AFTER=2\n", + (*AROUND, "OPTS"), + ), + "key_wrapped_below_a_joined_line": ( + f"A_BEFORE=1\nexport LONG={FAKE_KEY_LINE}\\\n{FAKE_KEY_LINE}\nFAKEt4il=\nZ_AFTER=2\n", + (*AROUND, "LONG"), + ), + "line_separator_in_a_value": ("A_BEFORE=1\nKEY=abc\u2028FAKEls=\nZ_AFTER=2\n", (*AROUND, "KEY")), + "form_feed_in_a_value": ("A_BEFORE=1\nKEY=abc\x0cFAKEff=\nZ_AFTER=2\n", (*AROUND, "KEY")), + "byte_order_mark": ("\ufeffOPENAI_API_KEY=\nZ_AFTER=2\n", ("OPENAI_API_KEY", "Z_AFTER")), + "escaped_backslash_before_the_closing_quote": ( + 'A_BEFORE=1\nWIN="C:\\\\tmp\\\\"\nMID_NAME=3\nZ_AFTER=2\n', + (*AROUND, "WIN", "MID_NAME"), + ), + "key_marker_in_an_inline_comment": ( + "A_BEFORE=1\nKEY=x # paste -----BEGIN RSA PRIVATE KEY----- here\nMID_NAME=3\nZ_AFTER=2\n", + (*AROUND, "KEY", "MID_NAME"), + ), + "key_marker_in_a_comment_line": ( + "A_BEFORE=1\n# paste the key from -----BEGIN on\nMID_NAME=3\nZ_AFTER=2\n", + (*AROUND, "MID_NAME"), + ), + "one_line_key_block": ( + "A_BEFORE=1\nKEY=-----BEGIN FAKE KEY-----FAKEbody-----END FAKE KEY-----\nMID_NAME=3\nZ_AFTER=2\n", + (*AROUND, "KEY", "MID_NAME"), + ), + "word_names": ( + "apiKey=\njwtSecret=\nOPENAIAPIKEY=\nlowercasename=\nLANGCHAIN_TRACING_V2=\n" + "NEXT_PUBLIC_SUPABASE_ANON_KEY=\n", + ( + "apiKey", + "jwtSecret", + "OPENAIAPIKEY", + "lowercasename", + "LANGCHAIN_TRACING_V2", + "NEXT_PUBLIC_SUPABASE_ANON_KEY", + ), + ), + "export_quoted": ( + f'A_BEFORE=1\nexport KEY="{FAKE_KEY_LINE}\nFAKEex=\n"\nZ_AFTER=2\n', + (*AROUND, "KEY"), + ), + "inline_comment_with_a_quote": ( + "A_BEFORE=1\nKEY=plain # don't\nMID_NAME=3\nKEY2=\"v\" # it's\nZ_AFTER=2\n", + (*AROUND, "KEY", "MID_NAME", "KEY2"), + ), + "key_block_marked": ( + f"A_BEFORE=1\nPEM=-----BEGIN PRIVATE KEY-----\n{FAKE_KEY_LINE}\nFAKEpm=\n" + "-----END PRIVATE KEY-----\nZ_AFTER=2\n", + (*AROUND, "PEM"), + ), + "json_blob_over_lines": ( + 'A_BEFORE=1\nSA={\n "type": "service_account",\n "private_key_id": "FAKEid",\n' + ' "private_key": "-----BEGIN PRIVATE KEY-----\\nFAKEjs=\\n-----END PRIVATE KEY-----\\n"\n' + "}\nZ_AFTER=2\n", + (*AROUND, "SA"), + ), + "json_blob_on_one_line": ( + 'A_BEFORE=1\nSA={"k":"FAKEjc=","p":"FAKEjc2="}\nZ_AFTER=2\n', + (*AROUND, "SA"), + ), + "crlf_quoted": ( + f'A_BEFORE=1\r\nKEY="{FAKE_KEY_LINE}\r\nFAKEcr=\r\n"\r\nZ_AFTER=2\r\n', + (*AROUND, "KEY"), + ), + "backtick_over_two_lines": ( + f"A_BEFORE=1\nKEY=`{FAKE_KEY_LINE}\nFAKEbt=`\nZ_AFTER=2\n", + (*AROUND, "KEY"), + ), + "backtick_over_three_lines": ( + f"A_BEFORE=1\nKEY=`{FAKE_KEY_LINE}\n{FAKE_KEY_LINE}\nFAKEbt=`\nMID_NAME=3\nZ_AFTER=2\n", + (*AROUND, "KEY", "MID_NAME"), + ), + "escaped_backtick": ("A_BEFORE=1\nESC=`say \\`hi\nFAKEebt=\n`\nZ_AFTER=2\n", (*AROUND, "ESC")), + "encoded_line_ending_in_a_blank": ( + f"A_BEFORE=1\nKEY={FAKE_KEY_LINE} \nFAKEtb=\nZ_AFTER=2\n", + (*AROUND, "KEY"), + ), + "encoded_line_ending_in_a_tab": ( + f"A_BEFORE=1\nKEY={FAKE_KEY_LINE}\t\nFAKEtt=\nZ_AFTER=2\n", + (*AROUND, "KEY"), + ), + "encoded_line_ending_in_a_comment": ( + f"A_BEFORE=1\nKEY={FAKE_KEY_LINE} # paste the key here\nFAKEtc=\nZ_AFTER=2\n", + (*AROUND, "KEY"), + ), +} + + +def _name_problems(names: object, expected: tuple[str, ...]) -> tuple[list[int], list[str]]: + """The lengths of the names that shouldn't be there, and the names missing. Lengths only, + so a failure never prints a piece of a value.""" + found = set(names) # type: ignore[call-overload] + return sorted(len(name) for name in found - set(expected)), sorted(set(expected) - found) + + +@pytest.mark.parametrize("shape", sorted(TEMPLATE_SHAPES)) +def test_a_template_shape_gives_its_names_and_no_value(shape: str) -> None: + text, expected = TEMPLATE_SHAPES[shape] + + problems = _name_problems(scan_walk._template_names(text.encode("utf-8")), expected) + + assert problems == ([], []) + + +@pytest.mark.parametrize( + "encoded", + [ + lambda text: b"\xff\xfe" + text.encode("utf-16-le"), + lambda text: b"\xfe\xff" + text.encode("utf-16-be"), + lambda text: text.encode("latin-1"), + ], + ids=["utf16_little_endian", "utf16_big_endian", "latin1"], +) +def test_a_template_in_utf16_or_latin1_gives_its_names_and_no_value( + tmp_path: Path, encoded: object +) -> None: + # Not UTF-8 either way: a byte-order mark of UTF-16, or a Latin-1 `ü`. + text = ( + f"A_BEFORE=1\nCITY=Zürich\nSIGNING_KEY={FAKE_KEY_LINE}\n{FAKE_KEY_LINE}\nFAKEt4il=\n" + "Z_AFTER=2\n" + ) + expected = ("A_BEFORE", "CITY", "SIGNING_KEY", "Z_AFTER") + data = encoded(text) # type: ignore[operator] + (tmp_path / ".env.example").write_bytes(data) + (tmp_path / "bot.py").write_text(CLIENT, encoding="utf-8") + + problems = _name_problems(scan_walk._template_names(data), expected) + walked = _walk(tmp_path) + walked_problems = _name_problems(walked.agents[0].env_names, expected) + + assert problems == ([], []) + assert walked.unread_files == 0 + assert walked_problems == ([], []) + + +def test_a_multi_line_value_in_a_template_is_never_a_name(tmp_path: Path) -> None: + # Every line of a pasted key below carries "Fake"; the last base64 line of a key ends in + # `=` padding, so it reads like a setting with an empty value. + template = ( + "BEFORE_NAME=1\n" + 'QUOTED_KEY="-----BEGIN PRIVATE KEY-----\n' + "MIIEvQIBADANBgkqhkiG9w0BAQEFAASCFake01\n" + "QuotedTailFake02=\n" + "-----END PRIVATE KEY-----\"\n" + "MIDDLE_NAME=2\n" + "RAW_KEY=-----BEGIN PRIVATE KEY-----\n" + "MIIEvQIBADANBgkqhkiG9w0BAQEFAASCFake03\n" + "RawTailFake04=\n" + "-----END PRIVATE KEY-----\n" + "SINGLE_QUOTED='first line\n" + "SingleTailFake05=\n" + "'\n" + "BarePaddedFake06==\n" + "WRAPPED_BLOB=MIIBIjANBgkqhkiG9w0BAQEFAAOCFake07\n" + "hkiG9w0BAQEFAAOCAQ8AFake08=\n" + "AFTER_NAME=3\n" + "OPENAIAPIKEY=\n" + "lowercasename=\n" + ) + expected = ( + "AFTER_NAME", + "BEFORE_NAME", + "MIDDLE_NAME", + "OPENAIAPIKEY", + "QUOTED_KEY", + "RAW_KEY", + "SINGLE_QUOTED", + "WRAPPED_BLOB", + "lowercasename", + ) + # Each shape in a repository of its own, so one shape's value can't hide another's. + repos = {"mixed": (template, expected), **TEMPLATE_SHAPES} + for shape, (text, _) in repos.items(): + (tmp_path / shape).mkdir() + (tmp_path / shape / ".env.example").write_bytes(text.encode("utf-8")) + (tmp_path / shape / "bot.py").write_text(CLIENT, encoding="utf-8") + + for shape, (_, names) in repos.items(): + [agent] = _walk(tmp_path / shape).agents + described = describe_entry(tmp_path / shape, "bot.py", "", deadline=FOREVER) + + assert described is not None + for found in (agent.env_names, described.env_names): + problems = _name_problems(found, names) + assert (shape, problems) == (shape, ([], [])) + assert found == tuple(sorted(names)) + assert not any("FAKE" in name.upper() or "MIIE" in name for name in found), shape + + +def test_the_walks_hazard_look_stops_at_the_walks_deadline( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # The build counts `agent.py` in `venv/`, which the walk itself skips. + _write(tmp_path, {"bot.py": CLIENT, "venv/a/agent.py": "", "venv/b/agent.py": ""}) + [on_time] = _walk(tmp_path).agents + assert on_time.build_may_stop == "agent_py" + listed: list[str] = [] + real_walk = os.walk + + def counting(top, *args, **kwargs): + for directory, directories, files in real_walk(top, *args, **kwargs): + listed.extend(os.path.join(directory, name) for name in directories + files) + yield directory, directories, files + + monkeypatch.setattr(scan_walk.os, "walk", counting) + # On time while the walk reads its one file, past the deadline from then on. + ticks = iter([0.0]) + + [late] = walk_repo(tmp_path, deadline=10.0, clock=lambda: next(ticks, 100.0)).agents + + assert (late.entry, late.build_may_stop) == ("bot.py", "") + assert not any(path.endswith("agent.py") for path in listed) + + +def test_build_hazard_is_bounded_and_asked_once_per_folder( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + assert scan_walk._HAZARD_ENTRIES == 200_000 + _write(tmp_path, {f"bot_{index}.py": CLIENT for index in range(5)}) + for index in range(30): + (tmp_path / "venv" / f"pkg_{index:02d}").mkdir(parents=True) + entries: list[str] = [] + real_walk = os.walk + + def counting(top, *args, **kwargs): + for directory, directories, files in real_walk(top, *args, **kwargs): + entries.extend(os.path.join(directory, name) for name in directories + files) + yield directory, directories, files + + monkeypatch.setattr(scan_walk.os, "walk", counting) + venv = os.path.join(tmp_path, "venv") + os.sep + + hazards: dict[str, str] = {} + result = walk_repo(tmp_path, deadline=FOREVER, hazards=hazards) + for _ in range(2): + describe_entry(tmp_path, "bot_0.py", "", deadline=FOREVER, hazards=hazards) + + # Five root agents and two descriptions share the root: its skipped `venv/` is looked + # through once. + assert len(result.agents) == 5 + assert hazards == {"": ""} + assert sum(path.startswith(venv) for path in entries) == 30 + + # Two `agent.py` files, each 15 folders down: finding both visits about 30 entries. + for branch in ("a", "b"): + _write(tmp_path, {"/".join(["deep", *[branch] * 15, "agent.py"]): ""}) + entries.clear() + monkeypatch.setattr(scan_walk, "_HAZARD_ENTRIES", 10) + assert build_hazard(tmp_path, "deep") == "" + assert len(entries) <= 11 + + monkeypatch.setattr(scan_walk, "_HAZARD_ENTRIES", 200_000) + assert build_hazard(tmp_path, "deep") == "agent_py" + + entries.clear() + assert build_hazard(tmp_path, "deep", deadline=0.0, clock=lambda: 1.0) == "" + # The folder's own listing and the one the clock is checked after, never the rest. + assert len(entries) <= 3 + + +def test_the_prompt_files_listed_are_capped( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + assert scan_walk._MAX_PROMPT_PATHS == 20_000 + _write(tmp_path, {"bot.py": CLIENT, **{f"prompts/p{index}.txt": "x\n" for index in range(5)}}) + monkeypatch.setattr(scan_walk, "_MAX_PROMPT_PATHS", 3) + + [agent] = _walk(tmp_path).agents + + assert agent.prompt_files == 3 + + +def _decision_table_sources(row: str) -> tuple[str, ...]: + """One row's files from the framework check's own decision table, read from its test.""" + tree = ast.parse((Path(__file__).parent / "test_framework_detection.py").read_text()) + test = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "test_decision_table" + ) + table = test.decorator_list[0] + assert isinstance(table, ast.Call) + ids = next(keyword.value for keyword in table.keywords if keyword.arg == "ids") + index = ast.literal_eval(ids).index(row) + rows = table.args[1] + assert isinstance(rows, ast.List) + sources, _ = rows.elts[index].elts # type: ignore[attr-defined] + return ast.literal_eval(sources) + + +def test_a_chat_model_alone_is_a_candidate(tmp_path: Path) -> None: + [source] = _decision_table_sources("model_alone") + _write(tmp_path, {"bot.py": source}) + + result = _walk(tmp_path) + + [agent] = result.agents + assert (agent.entry, agent.sign, agent.builds) == ("bot.py", "model", ()) + assert result.candidate_outside_side + assert ("model langchain", "bot.py", 2) in agent.evidence + + +def test_a_chat_model_beside_a_stronger_agent_is_not_listed(tmp_path: Path) -> None: + chat_model = "from langchain_openai import ChatOpenAI\nChatOpenAI()\n" + _write( + tmp_path, + { + "svc/pyproject.toml": "", + "svc/graph.py": GRAPH, + "svc/helper.py": chat_model, + "other/pyproject.toml": "", + "other/chain.py": chat_model, + }, + ) + + result = _walk(tmp_path) + + assert {agent.entry: agent.sign for agent in result.agents} == { + "svc/graph.py": "framework", + "other/chain.py": "model", + } + + +def test_a_deeper_project_does_not_move_an_agents_folder(tmp_path: Path) -> None: + """The platform keys an agent on repo and subfolder, so adding an unrelated project with + its own agent must not move the first agent's folder.""" + project = "[project]\nname = 'x'\n" + _write(tmp_path, {"pyproject.toml": project, "src/app/agent.py": GRAPH}) + alone = _walk(tmp_path) + assert {agent.entry: agent.subfolder for agent in alone.agents} == {"src/app/agent.py": ""} + + _write(tmp_path, {"plugins/x/pyproject.toml": project, "plugins/x/agent.py": GRAPH}) + beside = _walk(tmp_path) + assert {agent.entry: agent.subfolder for agent in beside.agents} == { + "src/app/agent.py": "", + "plugins/x/agent.py": "plugins/x", + } + + +def test_side_folders_cannot_use_up_the_file_cap_before_the_agent( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The customer's own code is read first; a big tests folder sorted ahead of the agent + no longer hides it behind the file cap.""" + monkeypatch.setattr(scan_walk, "_MAX_FILES", 10) + _write(tmp_path, {f"tests/test_{index:02}.py": "x = 1\n" for index in range(25)}) + _write(tmp_path, {"zz_agent/agent.py": GRAPH}) + + result = _walk(tmp_path) + + assert "zz_agent/agent.py" in _entries(result) + assert result.stopped == "limit" + + +@pytest.mark.parametrize("shape", ["linked_template", "unterminated_py", "long_line_py", "over_size_py"]) +def test_one_unreadable_file_does_not_stop_an_ai_rows_description(tmp_path: Path, shape: str) -> None: + """The row was paid for: a bad file marks it partial, and the reading goes on.""" + _write( + tmp_path, + { + "app/bot.py": GRAPH + "import os\nos.getenv('OPENAI_API_KEY')\n", + "app/z_tool.py": "import os\nos.getenv('BOT_MODEL')\n", + }, + ) + if shape == "linked_template": + (tmp_path / "outside.env").write_text("SEARCH_KEY=x\n", encoding="utf-8") + (tmp_path / "app/.env.example").symlink_to(tmp_path / "outside.env") + elif shape == "unterminated_py": + (tmp_path / "app/a_gen.py").write_text("x = '''never closed\n", encoding="utf-8") + elif shape == "long_line_py": + (tmp_path / "app/a_gen.py").write_text("x = 1 # " + "y" * 70_000 + "\n", encoding="utf-8") + else: + (tmp_path / "app/a_gen.py").write_text("x = 1\n" * 300_000, encoding="utf-8") + + candidate = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) + + assert candidate is not None + assert candidate.partial is True + assert {"OPENAI_API_KEY", "BOT_MODEL"} <= set(candidate.env_names) + assert "langgraph" in candidate.builds + + +def test_every_file_is_read_in_exactly_one_pass(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The two passes split the files: together they give each file once.""" + _write( + tmp_path, + { + "agent.py": GRAPH, + "app/.env.example": "A=1\n", + "app/test_helpers.py": "x = 1\n", + "app/conftest.py": "x = 1\n", + "tests/test_a.py": "x = 1\n", + "tests/.env.example": "B=1\n", + "examples/demo/agent.py": GRAPH, + "pkg/sub/mod.py": "x = 1\n", + "pkg/sub/tests/test_b.py": "x = 1\n", + "zz_app/.env.example": "C=1\n", + }, + ) + seen: list[str] = [] + take = scan_walk._Walk._take + + def counting(self, directory, folder, name): + seen.append(scan_walk._join(folder, name)) + return take(self, directory, folder, name) + + monkeypatch.setattr(scan_walk._Walk, "_take", counting) + _walk(tmp_path) + + every = sorted(p.relative_to(tmp_path).as_posix() for p in tmp_path.rglob("*") if p.is_file()) + assert sorted(seen) == every + assert len(seen) == len(set(seen)) + # A settings template is read with its folder's own code, before every side file: here + # one in a folder that sorts after tests/, which a side-pass template would follow. + assert seen.index("zz_app/.env.example") < seen.index("tests/test_a.py") + assert seen.index("app/.env.example") < seen.index("app/test_helpers.py") + + +def test_a_template_beside_the_agent_is_read_before_a_big_tests_folder( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(scan_walk, "_MAX_FILES", 10) + _write(tmp_path, {f"tests/test_{index:02}.py": "x = 1\n" for index in range(25)}) + _write(tmp_path, {"zz_agent/agent.py": GRAPH, "zz_agent/.env.example": "OPENAI_API_KEY=\n"}) + + result = _walk(tmp_path) + + assert "OPENAI_API_KEY" in _agent(result, "zz_agent/agent.py").env_names + + +def test_a_reader_fault_on_every_file_logs_one_trace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + _write(tmp_path, {f"m{index}.py": "x = 1\n" for index in range(5)}) + + def broken(data: bytes, shadowed: object) -> object: + raise LookupError("a fault in the kit's reader") + + monkeypatch.setattr(scan_walk, "_python_facts", broken) + with caplog.at_level("WARNING", logger=scan_walk.logger.name): + result = _walk(tmp_path) + + assert result.unread_files == 5 + assert caplog.text.count("a file's reader failed") == 1 + assert "LookupError" in caplog.text + + +def test_a_reader_fault_on_every_described_file_logs_one_trace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + _write(tmp_path, {"app/bot.py": GRAPH, **{f"app/m{index}.py": "x = 1\n" for index in range(5)}}) + + def broken(data: bytes, shadowed: object) -> object: + raise LookupError("a fault in the kit's reader") + + monkeypatch.setattr(scan_walk, "_python_facts", broken) + with caplog.at_level("WARNING", logger=scan_walk.logger.name): + candidate = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) + + assert candidate is not None and candidate.partial is True + assert caplog.text.count("a file's reader failed") == 1 + + +def test_the_step_after_the_walk_watches_the_clock(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Dileep's shape, small: many files that build a client, each importing a web of modules. + Joining them into agents must stop at its deadline and end as a clock stop, never run on + to the helper's hard stop.""" + files = { + f"pkg/m{index}.py": "".join(f"from .m{(index * 7 + step) % 200} import x\n" for step in range(1, 6)) + for index in range(200) + } + files.update( + { + f"pkg/c{index}.py": CLIENT + "".join(f"from .m{(index * 3 + step) % 200} import x\n" for step in range(5)) + for index in range(50) + } + ) + _write(tmp_path, files) + + complete = _walk(tmp_path) + assert complete.stopped == "" and complete.agents + + monkeypatch.setattr(scan_walk, "_RESULT_GRACE", 0.0) + calls: list[int] = [] + + def clock() -> float: + calls.append(1) + return float(len(calls)) + + # Each file read checks the clock once, so this deadline lets the walk finish and then + # gives the join only a few checks. + timed = walk_repo(tmp_path, deadline=float(len(files) + 5), clock=clock) + + assert timed.stopped == "clock" + assert timed.agents == () + assert len(calls) < len(files) + 50 + + +def test_the_step_after_the_walk_holds_a_bounded_number_of_edges( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Dileep's shape, small: every candidate imports one package that re-exports modules, so + each candidate reaches every other and the pairs grow with the square of the candidates.""" + files = {"pkg/__init__.py": "".join(f"from .m{index} import x\n" for index in range(20))} + files.update({f"pkg/m{index}.py": "x = 1\n" for index in range(20)}) + files.update({f"c{index}.py": CLIENT + "import pkg\nfrom pkg import x\n" for index in range(30)}) + files.update({f"pkg/m{index}.py": CLIENT for index in range(20, 40)}) + files["pkg/__init__.py"] += "".join(f"from .m{index} import x\n" for index in range(20, 40)) + _write(tmp_path, files) + + assert _walk(tmp_path).stopped == "" + + monkeypatch.setattr(scan_walk, "_MAX_EDGES", 10) + bounded = _walk(tmp_path) + + assert bounded.stopped == "limit" + assert bounded.agents == () diff --git a/tests/harness/test_scenario_source_provision_agent.py b/tests/harness/test_scenario_source_provision_agent.py index 9cf5e53a9..49a4fc21e 100644 --- a/tests/harness/test_scenario_source_provision_agent.py +++ b/tests/harness/test_scenario_source_provision_agent.py @@ -126,6 +126,25 @@ def test_provision_payload_carries_the_agent_built_from_the_sealed_contract( ) +def test_a_url_urlsplit_refuses_is_registered_redacted_and_logged_by_class_only( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + # A full-width number sign NFKC-normalises to "#", so urlsplit refuses the netloc, and its + # error repeats the whole userinfo. + leaky = "https://admin:hunter2\uff03x@db.example/" + _write_bundle(tmp_path, _contract(leaky)) + + with caplog.at_level(logging.DEBUG): + payload = _provision(tmp_path, SourceKind.GITHUB) + + assert payload["agent"]["agent"]["behaviour"][0]["instructions"] == "[redacted]" + assert "hunter2" not in json.dumps(payload["agent"]) + [logged] = [record for record in caplog.records if record.name == "fi.alk.providers.agent"] + assert logged.getMessage() == "a URL was redacted whole: class=ValueError" + assert logged.exc_info is None + assert "hunter2" not in caplog.text + + @pytest.mark.parametrize( ("kind", "contract", "build_error", "logged"), [