From 519f8189977f211d68d43691dee0202a99aeb03f Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Thu, 5 Nov 2026 11:57:52 +0530 Subject: [PATCH 01/23] feat(harness): read an agent without building it - Add --read-agent-only to the hosted authoring command: run only the agent reading and leave contract.json, agent.json, cost.json and any failure.json in --output, then stop. - Share one helper, reading_entry, with the agent-finding mode: job check, a soft limit and a hard timer, masking of secrets files, and a cost file that starts incomplete and is rewritten after each turn. - Count Claude's running totals once per turn, keep a contract the gate saved over any later fault, and refuse browser agents by the same rule a build uses. - Treat a turn limit as final, never retried: an error outcome, or ADK's whole call-limit message. A status-less provider failure is retried and marks the cost incomplete. TH-8951 Signed-off-by: khushalsonawat --- .github/workflows/sdk-smoke.yml | 2 + src/fi/alk/harness/agent_reading.py | 566 +++++++ src/fi/alk/harness/authoring_entrypoint.py | 15 + src/fi/alk/harness/reading_entry.py | 253 ++++ tests/harness/test_agent_reading.py | 1539 ++++++++++++++++++++ tests/harness/test_reading_entry.py | 536 +++++++ 6 files changed, 2911 insertions(+) create mode 100644 src/fi/alk/harness/agent_reading.py create mode 100644 src/fi/alk/harness/reading_entry.py create mode 100644 tests/harness/test_agent_reading.py create mode 100644 tests/harness/test_reading_entry.py diff --git a/.github/workflows/sdk-smoke.yml b/.github/workflows/sdk-smoke.yml index 851f7eb53..9777af0dd 100644 --- a/.github/workflows/sdk-smoke.yml +++ b/.github/workflows/sdk-smoke.yml @@ -102,6 +102,8 @@ jobs: tests/harness/test_instrumentation_evidence.py tests/harness/test_agent_type_refusal.py tests/harness/test_executor_completed_counts.py + tests/harness/test_reading_entry.py + tests/harness/test_agent_reading.py tests/test_bump_deployment_values.py tests/test_release_workflow.py tests/test_release_pr_step.py diff --git a/src/fi/alk/harness/agent_reading.py b/src/fi/alk/harness/agent_reading.py new file mode 100644 index 000000000..1678aa96a --- /dev/null +++ b/src/fi/alk/harness/agent_reading.py @@ -0,0 +1,566 @@ +"""Read one agent the way a build reads it, and leave its description, with no build after it. + +The platform shows an agent's description before any environment exists, and bills the reading +from ``cost.json``. So the reading runs the build's own stage and gate, and every way it can end +leaves the files the platform judges it by: a saved contract is never thrown away and paid for +again, a timeout is never retried as a crash, and the cost on disk is never marked complete +while some of the spend may be missing from it. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import math +import os +import re +import tempfile +import time +from dataclasses import replace +from pathlib import Path +from typing import Any + +from .. import __version__ +from ..providers.agent import AgentBuildError, build_authored_agent +from . import session +from .backends import StageDone +from .builder import BUILDER_TURNS +from .config import chosen_model +from .contract import AgentContract +from .failure_file import FAILURE_FILE +from .framework_detection import _contract_workdir, detect_framework +from .job import FailureDomain, HarnessFailure, HarnessJob, HarnessStage, SourceKind +from .reading_entry import ( + READING_JOB_INVALID, + ReadingMode, + write_reading_cost, + write_reading_failure, +) +from .session import Stage, StageIdleTimeout +from .source_folder import agent_folder, refuse_unusable_folder +from .sources import resolve as resolve_source +from .understand import open_stage, opening + +logger = logging.getLogger(__name__) + +AGENT_FILE = "agent.json" +CONTRACT_FILE = "contract.json" + +READING_SOURCE_EMPTY = "reading_source_empty" +AGENT_READING_TIMED_OUT = "agent_reading_timed_out" +AGENT_READING_CRASHED = "agent_reading_crashed" +AGENT_READING_MODEL_ERROR = "agent_reading_model_error" +AGENT_READING_INCOMPLETE = "agent_reading_incomplete" +AGENT_READING_TOO_LARGE = "agent_reading_too_large" +AGENT_DOCUMENT_INVALID = "agent_document_invalid" + +_LIMIT_ENV = "ALK_READING_MAX_DURATION_SECONDS" +_DEFAULT_SOFT_LIMIT_SECONDS = 1800.0 +# The hard limit outlasts the soft one by the interrupt, the wait after it, and a margin, so a +# reading that stops itself is never cut off by the helper first. +_HARD_LIMIT_MARGIN_SECONDS = 180.0 +_INTERRUPT_SECONDS = 10.0 +_INTERRUPT_WAIT_SECONDS = 60.0 + +# The platform's own caps, compared the way it compares them. +_DOCUMENT_LIMIT_BYTES = 1_048_576 +_CONTRACT_LIMIT_BYTES = 5 * 1_048_576 + +# A model out of turns can't save, so a reminder after one only adds cost. +_TURN_LIMITS = frozenset({"max_turns", "error_max_turns"}) +# Matched only, never repeated in a message: these are ADK's words, not ours. +_CALL_LIMIT = re.compile(r"Max number of llm calls limit of `\d+` exceeded") +# Statuses a retry may get past; any other 4xx would only be refused again. +_PASSING_STATUSES = frozenset({408, 409, 429}) +_COUNTS = ("tokens_in", "tokens_out", "tokens_cached") + +_EXIT_DONE = 0 +_EXIT_RETRYABLE = 1 +_EXIT_REFUSED = 78 + +# The same words the build's unattended reading uses, so the two readings can't drift apart. +_REMINDER = ( + "Nothing was saved: you finished without calling submit_contract. Call it now " + "with the contract you worked out." +) + + +def _soft_limit() -> float: + try: + seconds = float(os.environ.get(_LIMIT_ENV, "")) + except ValueError: + return _DEFAULT_SOFT_LIMIT_SECONDS + return seconds if math.isfinite(seconds) and seconds > 0 else _DEFAULT_SOFT_LIMIT_SECONDS + + +def _failure( + domain: FailureDomain, + code: str, + message: str, + *, + retryable: bool, + details: dict[str, Any] | None = None, +) -> HarnessFailure: + return HarnessFailure( + domain=domain, + stage=HarnessStage.UNDERSTANDING_AGENT, + code=code, + message=message, + retryable=retryable, + details=details or {}, + ) + + +def _check_job(job: HarnessJob) -> HarnessFailure | None: + if job.source.kind in (SourceKind.GITHUB, SourceKind.ARCHIVE): + return None + return _failure( + FailureDomain.PLATFORM_SYNC, + READING_JOB_INVALID, + f"An agent is read from a GitHub repository or an upload, not from a " + f"{job.source.kind.value} source.", + retryable=False, + ) + + +class _ReadingStage(Stage): + """The build's reading stage, with each turn's own spend kept and written as it lands.""" + + def __init__( + self, + spec: Any, + *, + name: str, + backend: Any, + output: Path, + running_totals: bool, + ) -> None: + super().__init__(spec, name=name, backend=backend) + self.output = output + self.running_totals = running_totals + self.done: list[StageDone] = [] + self.in_flight = False + self.opened = False + self._last_cost = 0.0 + self._last_counts = dict.fromkeys(_COUNTS, 0) + + async def __aenter__(self) -> "_ReadingStage": + await super().__aenter__() + self.opened = True + return self + + def _own(self, reported: StageDone) -> StageDone: + # Claude reports the session's running total on every result, so adding results as + # reported would count every earlier turn again. + if not self.running_totals: + return reported + cost = reported.cost_usd + if cost is not None: + cost, self._last_cost = max(0.0, cost - self._last_cost), cost + counts: dict[str, int] = {} + for key in _COUNTS: + total = int(getattr(reported, key) or 0) + counts[key] = max(0, total - self._last_counts[key]) + self._last_counts[key] = total + return replace(reported, cost_usd=cost, **counts) + + def _events(self, received: Any, turn: Any) -> list[Any]: + if not isinstance(received, StageDone): + return super()._events(received, turn) + done = self._own(received) + events = super()._events(done, turn) + self.done.append(done) + self.in_flight = False + # On disk after every turn, so a sandbox that dies mid-reading still leaves what it spent. + write_reading_cost(self.output, complete=False) + return events + + async def say(self, message: str, **kwargs: Any) -> Any: + self.in_flight = True + return await super().say(message, **kwargs) + + +def _holds_anything(source: Path) -> bool: + try: + return source.is_dir() and next(source.iterdir(), None) is not None + except OSError: + # Read as empty, so retryable: a source the sandbox can't list yet may be listable next time. + return False + + +def _saved(output: Path) -> str: + if (output / FAILURE_FILE).is_file(): + return FAILURE_FILE + if (output / CONTRACT_FILE).is_file(): + return CONTRACT_FILE + return "" + + +def _out_of_turns(done: StageDone) -> bool: + # ADK enforces its call budget by raising, which the Gemini backend reports as a failure with + # no status, so ADK's own words are the signal. Matched whole, since the phrase inside other + # text may be the model's own. The count is no signal: it catches no real limit, and it would + # read a request cut on the last allowed call as one. + return done.outcome in _TURN_LIMITS or ( + done.is_error + and done.api_error_status is None + and any(_CALL_LIMIT.fullmatch(str(said)) for said in done.errors) + ) + + +def _reminder_due(stage: _ReadingStage, output: Path, stopping: asyncio.Event) -> bool: + if _saved(output) or stopping.is_set(): + return False + opened = stage.done[-1] if stage.done else None + # An error flag with no status may have passed; one nudge costs less than a whole retry. + return opened is None or ( + opened.api_error_status is None and not _out_of_turns(opened) + ) + + +async def _converse( + stage: _ReadingStage, agent_source: Any, output: Path, stopping: asyncio.Event +) -> None: + # The session lives inside this task, so cancelling the task closes it. + async with stage: + await stage.say(opening(agent_source)) + if _reminder_due(stage, output, stopping): + await stage.say(_REMINDER) + + +async def _stop(stage: _ReadingStage, task: asyncio.Task[None], stopping: asyncio.Event) -> None: + # Set first, so a turn ending now doesn't earn the reminder. + stopping.set() + try: + await asyncio.wait_for(stage.interrupt_response(), _INTERRUPT_SECONDS) + except Exception: + # A stop that can't be delivered is still a timeout, not a crash. + pass + await asyncio.wait({task}, timeout=_INTERRUPT_WAIT_SECONDS) + # Closed before the files are read, so no late tool call can save after the decision. + task.cancel() + try: + await task + except (asyncio.CancelledError, Exception): + pass + + +def _refusing(status: Any) -> bool: + return ( + isinstance(status, int) + and not isinstance(status, bool) + and 400 <= status <= 499 + and status not in _PASSING_STATUSES + ) + + +def _model_error(output: Path, message: str, status: Any = None, *, retryable: bool) -> int: + write_reading_failure( + _failure( + FailureDomain.INFRASTRUCTURE, + AGENT_READING_MODEL_ERROR, + message, + retryable=retryable, + # Only our own reading of the status: what the provider said stays in the message. + details={"status": status if isinstance(status, int) else None}, + ), + output, + ) + return _EXIT_RETRYABLE if retryable else _EXIT_REFUSED + + +def _timed_out(output: Path, soft_limit: float) -> int: + write_reading_failure( + _failure( + FailureDomain.INFRASTRUCTURE, + AGENT_READING_TIMED_OUT, + f"The agent reading ran past its {soft_limit:g} s limit without saving the agent's " + "description, and was stopped.", + retryable=False, + ), + output, + ) + return _EXIT_REFUSED + + +def _classify( + job: HarnessJob, + source: Path, + output: Path, + stage: _ReadingStage, + error: BaseException | None, +) -> int: + saved = _saved(output) + if saved == FAILURE_FILE: + return _EXIT_REFUSED + # Before any error: the gate accepted the reading, and refusing it now would pay for it twice. + if saved == CONTRACT_FILE: + return _document(job, source, output) + forbidden = stage.forbidden_models() + if forbidden: + # A retry would bill the same model again. + return _model_error( + output, + "The provider billed a model this harness may not spend on (" + + ", ".join(sorted(forbidden)) + + "), so the reading was stopped.", + retryable=False, + ) + # A last turn out of turns spent a whole budget, so a retry would likely spend another and + # stop the same way, whatever failed before it. + ran_out = bool(stage.done) and _out_of_turns(stage.done[-1]) + failed = [ + done + for done in stage.done + if done.api_error_status is not None + or (done.is_error and not _out_of_turns(done)) + ] + if not ran_out and (failed or error is not None): + status = next( + (done.api_error_status for done in reversed(failed) if done.api_error_status is not None), + None, + ) + said = [str(one) for done in failed for one in done.errors] + if error is not None: + said.append(f"{type(error).__name__}: {error}") + shown = f" (status {status})" if isinstance(status, int) else "" + return _model_error( + output, + f"The model provider failed while the agent was being read{shown}: " + + ("; ".join(said) or "no detail was given."), + status, + retryable=not _refusing(status), + ) + write_reading_failure( + _failure( + FailureDomain.INFRASTRUCTURE, + AGENT_READING_INCOMPLETE, + "The model finished without saving the agent's description.", + retryable=False, + details={ + "outcome": stage.done[-1].outcome if stage.done else "", + "turns": sum(int(done.turns or 0) for done in stage.done), + }, + ), + output, + ) + return _EXIT_REFUSED + + +class _TooLarge(Exception): + def __init__(self, sizes: dict[str, int]) -> None: + super().__init__("too large") + self.sizes = sizes + + +def _framework_folder(job: HarnessJob, source: Path, output: Path) -> Path: + # As a build chooses it when there is no build: the named folder, else the contract's own. + if job.source.subfolder: + return agent_folder(source, job.source.subfolder) + return _contract_workdir(source, output) or source + + +def _write_whole(path: Path, body: bytes) -> None: + # Written whole, then moved: the platform must never read half a document. + handle = tempfile.NamedTemporaryFile( + "wb", dir=path.parent, prefix=".agent-", suffix=".tmp", delete=False + ) + try: + with handle as writing: + writing.write(body) + os.replace(handle.name, path) + except BaseException: + Path(handle.name).unlink(missing_ok=True) + raise + + +def _document(job: HarnessJob, source: Path, output: Path) -> int: + """The agent's description from the saved contract, as a build makes it.""" + # One catch over every step: a saved contract is paid for, so nothing after it is retryable. + try: + contract_bytes = (output / CONTRACT_FILE).stat().st_size + if contract_bytes > _CONTRACT_LIMIT_BYTES: + raise _TooLarge({"contract_bytes": contract_bytes}) + contract = AgentContract.model_validate( + json.loads((output / CONTRACT_FILE).read_text(encoding="utf-8")) + ).model_dump(mode="json") + # The echo proves which folder was read, so it comes from the contract. + built_from = str((contract.get("runtime") or {}).get("source_subfolder") or "") + if built_from != (job.source.subfolder or ""): + raise AgentBuildError("subfolder_mismatch") + framework = detect_framework( + _framework_folder(job, source, output), repo_root=source, package_dir=None + ) + inner = build_authored_agent( + contract, subfolder=built_from or None, framework=framework + ) + body = json.dumps( + {"sdk_version": __version__, "agent": inner}, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + stored = json.dumps(inner, ensure_ascii=False).encode("utf-8") + if len(body) > _DOCUMENT_LIMIT_BYTES or len(stored) > _DOCUMENT_LIMIT_BYTES: + raise _TooLarge( + { + "agent_bytes": len(body), + "stored_bytes": len(stored), + "contract_bytes": contract_bytes, + } + ) + _write_whole(output / AGENT_FILE, body) + except _TooLarge as too_large: + write_reading_failure( + _failure( + FailureDomain.INFRASTRUCTURE, + AGENT_READING_TOO_LARGE, + "The agent's description is larger than the platform stores (1 MiB for the " + "description, 5 MiB for the contract).", + retryable=False, + details=dict(too_large.sizes), + ), + output, + ) + return _EXIT_REFUSED + except Exception as exc: + code = str(getattr(exc, "code", "") or "") + logger.warning( + "agent document not built: class=%s code=%s", type(exc).__name__, code + ) + write_reading_failure( + _failure( + FailureDomain.INFRASTRUCTURE, + AGENT_DOCUMENT_INVALID, + "The agent's description could not be built from its saved contract " + f"({type(exc).__name__}{': ' + code if code else ''}).", + retryable=False, + details={"class": type(exc).__name__, "code": code}, + ), + output, + ) + return _EXIT_REFUSED + return _EXIT_DONE + + +async def _reading( + job: HarnessJob, source: Path, output: Path, started: float, soft_limit: float +) -> tuple[int, bool]: + """The exit code, and whether every turn's spend is known to be in the cost file.""" + # Before the folder check, so an upload that hasn't arrived isn't blamed on the folder. + if not _holds_anything(source): + write_reading_failure( + _failure( + FailureDomain.PLATFORM_SYNC, + READING_SOURCE_EMPTY, + "The agent's source was missing or empty, so there was nothing to read. " + "It may not have arrived yet.", + retryable=True, + ), + output, + ) + return _EXIT_RETRYABLE, True + if refuse_unusable_folder(source, job.source.subfolder, output): + return _EXIT_REFUSED, True + + # A replayed turn is paid for again, and a new session would restart Claude's running + # totals under the baseline the difference is taken from. + session.STAGE_IDLE_RETRIES = 0 + backend = session.resolve() + agent_source = resolve_source( + "repo", + name=str(job.metadata.get("agent_name") or source.name), + root=source, + subfolder=job.source.subfolder, + ) + offered = job.metadata.get("available_evals") + opened, _ = open_stage( + agent_source, + out=output, + available_evals=offered if isinstance(offered, list) else None, + ) + # The build's settings, not the stage's own: the reading is meant to seed a build later. + stage = _ReadingStage( + replace(opened.spec, max_turns=BUILDER_TURNS, model=chosen_model(), thinking=False), + name=opened.name, + backend=backend, + output=output, + running_totals=getattr(backend, "name", "") == "claude", + ) + stopping = asyncio.Event() + task = asyncio.create_task(_converse(stage, agent_source, output, stopping)) + finished, _ = await asyncio.wait( + {task}, timeout=max(0.0, started + soft_limit - time.monotonic()) + ) + + if not finished: + await _stop(stage, task, stopping) + # A request cut off by the stop may be billed without being in the last total. + saved = _saved(output) + if saved == FAILURE_FILE: + return _EXIT_REFUSED, False + if saved == CONTRACT_FILE: + return _document(job, source, output), False + return _timed_out(output, soft_limit), False + + error = task.exception() + if error is not None and not stage.opened: + # Nothing reached the provider, so the fault is the reading's own: a crash, retried once. + raise error + if isinstance(error, StageIdleTimeout): + # The quiet turn's spend never arrived; what was saved before it still stands. + saved = _saved(output) + if saved == FAILURE_FILE: + return _EXIT_REFUSED, False + if saved == CONTRACT_FILE: + return _document(job, source, output), False + return ( + _model_error( + output, + "The model stopped responding before the agent's description was saved " + f"({error}).", + retryable=True, + ), + False, + ) + # A request cut with no status may still be billed, and its total never reached us. + complete = ( + error is None + and not stage.in_flight + and not any( + done.is_error and done.api_error_status is None and not _out_of_turns(done) + for done in stage.done + ) + ) + return _classify(job, source, output, stage, error), complete + + +async def _read(job: HarnessJob, source: Path, output: Path) -> int: + # The same instant the helper's hard limit counts from, give or take a few lines. + started = time.monotonic() + output = Path(output) + code, complete = _EXIT_RETRYABLE, False + try: + try: + code, complete = await _reading(job, Path(source), output, started, _soft_limit()) + except Exception as exc: + # A contract the gate saved is paid for; a crash would have the platform pay again. + if _saved(output) != CONTRACT_FILE: + raise + logger.warning("reading fault after a saved contract: class=%s", type(exc).__name__) + code, complete = _document(job, Path(source), output), False + finally: + # The work owns the final cost file, on every way out. + write_reading_cost(output, complete=complete) + return code + + +READ_AGENT = ReadingMode( + name="read_agent", + check_job=_check_job, + work=_read, + time_limit_seconds=_soft_limit() + _HARD_LIMIT_MARGIN_SECONDS, + timed_out_code=AGENT_READING_TIMED_OUT, + timed_out_retryable=False, + crashed_code=AGENT_READING_CRASHED, +) diff --git a/src/fi/alk/harness/authoring_entrypoint.py b/src/fi/alk/harness/authoring_entrypoint.py index e019b686f..07f4341e1 100644 --- a/src/fi/alk/harness/authoring_entrypoint.py +++ b/src/fi/alk/harness/authoring_entrypoint.py @@ -679,11 +679,26 @@ def main(argv: list[str] | None = None, *, validate_runtime: bool = False) -> in type=Path, help="Directory the sealed bundle is written to in --certify-only mode", ) + parser.add_argument( + "--read-agent-only", + action="store_true", + help="Read the agent and leave its description in --output; build nothing", + ) args = parser.parse_args(argv) if args.certify_only: if args.bundle_output is None: parser.error("--certify-only requires --bundle-output") return _certify_only(args) + if args.read_agent_only: + from .agent_reading import READ_AGENT + from .reading_entry import run_reading_mode + + return run_reading_mode( + READ_AGENT, + job_path=args.job, + source=args.source.resolve(), + output=args.output.resolve(), + ) job = HarnessJob.model_validate(json.loads(args.job.read_text(encoding="utf-8"))) if refuse_unusable_folder( diff --git a/src/fi/alk/harness/reading_entry.py b/src/fi/alk/harness/reading_entry.py new file mode 100644 index 000000000..d3ae470db --- /dev/null +++ b/src/fi/alk/harness/reading_entry.py @@ -0,0 +1,253 @@ +"""The one way into a reading of a repository: an agent read, or agents found, with no build. + +A reading is billed from ``cost.json`` and judged from ``failure.json``, so this module owns the +order in which those files appear. Bad input is refused before any model is paid for. The cost +file is on disk, at zero and marked incomplete, before the work starts, and only the work itself +may mark it complete. A hard time limit ends the process even when its own writes fail, because a +reading that outlives its sandbox leaves no file at all and is retried as a crash. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import sys +import tempfile +import threading +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +from . import spend +from .failure_file import write_failure_file +from .job import FailureDomain, HarnessFailure, HarnessJob, HarnessStage +from .outbound import redact_outbound_text + +READING_JOB_INVALID = "reading_job_invalid" +READING_OUTPUT_NOT_EMPTY = "reading_output_not_empty" +MESSAGE_LIMIT = 2000 +COST_FILE = "cost.json" + +_EXIT_RETRYABLE = 1 +_EXIT_REFUSED = 78 + +# Neither file is deleted by the hosted authoring entrypoint, so a read here takes nothing away +# from the code that loads them. +_SECRET_FILES = ( + Path("/run/futureagi/secrets.json"), + Path("/run/futureagi/simulator-secrets.json"), +) +_secret_values: tuple[str, ...] | None = None + + +@dataclass(frozen=True) +class ReadingMode: + name: str + check_job: Callable[[HarnessJob], HarnessFailure | None] + work: Callable[[HarnessJob, Path, Path], Awaitable[int]] + time_limit_seconds: float | None + timed_out_code: str + timed_out_retryable: bool + crashed_code: str + + +def _read_secret_values() -> tuple[str, ...]: + # Imported here: the module is large and the find-agents mode shares this helper. A broken + # import must not end a reading before its failure can be written, so it masks nothing extra. + try: + from .hosted_entrypoint import peek_secret_values + + return tuple(value for path in _SECRET_FILES for value in peek_secret_values(path)) + except Exception: + return () + + +def write_reading_failure(failure: HarnessFailure, output: Path) -> Path | None: + """Leave ``failure`` for the platform with its message made safe to show; the first one stands. + + Details are written as given, so a mode never puts what a provider said in them. + """ + try: + secrets = _secret_values if _secret_values is not None else _read_secret_values() + # Masked before the cut, so a secret split by the cut can't survive half-matched. + message = redact_outbound_text(failure.message, secrets)[:MESSAGE_LIMIT] + return write_failure_file(failure.model_copy(update={"message": message}), Path(output)) + except Exception: + return None + + +def write_reading_cost(output: Path, *, complete: bool) -> bool: + """Write the spend so far to ``cost.json``; ``True`` when it was written. + + ``complete`` is kept only in this file, never in ``spend.snapshot()``, which builds share. + """ + try: + output = Path(output) + output.mkdir(parents=True, exist_ok=True) + snapshot = spend.snapshot() + # A turn with no price makes the total a floor, and a floor billed as the total is wrong. + body = {**snapshot, "complete": bool(complete) and snapshot["unpriced_turns"] == 0} + # Written whole, then moved: a poll that reads mid-write must never see half a total. + handle = tempfile.NamedTemporaryFile( + "w", dir=output, prefix=".cost-", suffix=".tmp", encoding="utf-8", delete=False + ) + try: + with handle as writing: + json.dump(body, writing, indent=2, sort_keys=True) + os.replace(handle.name, output / COST_FILE) + except Exception: + Path(handle.name).unlink(missing_ok=True) + raise + except Exception: + # Accounting must never be the reason a reading fails. + return False + return True + + +def _failure( + domain: FailureDomain, code: str, message: str, *, retryable: bool +) -> HarnessFailure: + return HarnessFailure( + domain=domain, + stage=HarnessStage.UNDERSTANDING_AGENT, + code=code, + message=message, + retryable=retryable, + ) + + +def _holds_anything(output: Path) -> bool: + try: + if not os.path.lexists(output): + return False + return not output.is_dir() or next(output.iterdir(), None) is not None + except OSError: + # A folder that can't be listed can't be shown to be empty. + return True + + +def _start_hard_limit(mode: ReadingMode, output: Path) -> threading.Timer | None: + if mode.time_limit_seconds is None: + return None + exit_code = _EXIT_RETRYABLE if mode.timed_out_retryable else _EXIT_REFUSED + + def end_the_process() -> None: + try: + write_reading_failure( + _failure( + FailureDomain.INFRASTRUCTURE, + mode.timed_out_code, + f"The {mode.name} reading ran past its {mode.time_limit_seconds:g} s limit " + "and was stopped.", + retryable=mode.timed_out_retryable, + ), + output, + ) + write_reading_cost(output, complete=False) + finally: + # Work running in a thread can't be cancelled, and a normal exit would wait for it. + os._exit(exit_code) + + timer = threading.Timer(mode.time_limit_seconds, end_the_process) + timer.daemon = True + timer.start() + return timer + + +def run_reading_mode(mode: ReadingMode, *, job_path: Path, source: Path, output: Path) -> int: + """Refuse what can't be read, start the cost at zero, then run the mode's work.""" + global _secret_values + _secret_values = _read_secret_values() + output = Path(output) + + try: + job = HarnessJob.model_validate(json.loads(Path(job_path).read_text(encoding="utf-8"))) + except Exception as exc: + write_reading_failure( + _failure( + FailureDomain.PLATFORM_SYNC, + READING_JOB_INVALID, + f"The job file could not be read as a harness job ({type(exc).__name__}: {exc}).", + retryable=False, + ), + output, + ) + return _EXIT_REFUSED + + try: + refused = mode.check_job(job) + except Exception as exc: + # Nothing has been spent yet, so no cost file: a broken check is a crash, not a refusal. + write_reading_failure( + _failure( + FailureDomain.INFRASTRUCTURE, + mode.crashed_code, + f"{type(exc).__name__}: {exc}", + retryable=True, + ), + output, + ) + return _EXIT_RETRYABLE + if refused is not None: + write_reading_failure(refused, output) + return _EXIT_REFUSED + + # After the job checks on purpose: a stale folder is refused either way, and the failure + # they wrote into it is no less true. + if _holds_anything(output): + write_reading_failure( + _failure( + FailureDomain.PLATFORM_SYNC, + READING_OUTPUT_NOT_EMPTY, + "The reading's output folder already holds files; give it an empty folder.", + retryable=False, + ), + output, + ) + return _EXIT_REFUSED + + if not write_reading_cost(output, complete=False): + # A model paid for while its cost can't be recorded would read as nothing spent. + write_reading_failure( + _failure( + FailureDomain.INFRASTRUCTURE, + mode.crashed_code, + "The reading could not write its cost file, so it stopped before any model ran.", + retryable=True, + ), + output, + ) + return _EXIT_RETRYABLE + + timer = _start_hard_limit(mode, output) + crash: Exception | None = None + try: + print( + f"reading started at {datetime.now(timezone.utc).isoformat()}", + file=sys.stderr, + flush=True, + ) + code = asyncio.run(mode.work(job, source, output)) + except Exception as exc: + crash = exc + finally: + # Cancelled first, so the limit can't fire during a crash's writes or the caller's cleanup. + if timer is not None: + timer.cancel() + + if crash is not None: + write_reading_failure( + _failure( + FailureDomain.INFRASTRUCTURE, + mode.crashed_code, + f"{type(crash).__name__}: {crash}", + retryable=True, + ), + output, + ) + write_reading_cost(output, complete=False) + return _EXIT_RETRYABLE + # The work wrote its own final cost; rewriting it here could mark a lost turn as paid. + return code diff --git a/tests/harness/test_agent_reading.py b/tests/harness/test_agent_reading.py new file mode 100644 index 000000000..b649773d8 --- /dev/null +++ b/tests/harness/test_agent_reading.py @@ -0,0 +1,1539 @@ +"""Reading one agent with no build: the files it leaves, what it is charged, and how it stops. + +The model is a fake backend behind ``session.resolve``, so the real stage, the reading's own +stage, the contract server and the browser-agent gate all run; only the provider is scripted. +""" + +from __future__ import annotations + +import ast +import asyncio +import inspect +import json +import os +import shutil +import textwrap +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest + +import fi.alk +from fi.alk.harness import agent_reading, authoring_entrypoint, reading_entry, session, spend +from fi.alk.harness.backends import StageDone +from fi.alk.harness.builder import BUILDER_TURNS +from fi.alk.harness.config import chosen_model +from fi.alk.harness.contract import AgentContract +from fi.alk.harness.framework_detection import detect_framework +from fi.alk.harness.job import HarnessJob, SourceKind +from fi.alk.harness.reading_entry import READING_JOB_INVALID, run_reading_mode +from fi.alk.harness.tools import CONTRACT_SERVER +from fi.alk.providers.agent import build_authored_agent + +EXAMPLES = Path(__file__).resolve().parents[2] / "examples" / "frameworks" +SUPPORT = "agents/support" +SCRAPER = "agents/scraper" +MIB = 1_048_576 +CHAT = { + "agent": "support", + "modality": "chat", + "real_use_cases": ["order"], + "tools": [{"name": "add", "args": ["item_id"]}], +} +# As the gate saves it for a job that names the folder. +IN_SUPPORT = { + **CHAT, + "runtime": {"language": "python", "workdir": SUPPORT, "source_subfolder": SUPPORT}, +} +# The real one, kept before the fixture shortens the limit for every other test. +_SOFT_LIMIT = agent_reading._soft_limit + + +@pytest.fixture(autouse=True) +def _fresh_state(monkeypatch, tmp_path): + monkeypatch.setattr(session, "STAGE_IDLE_RETRIES", session.STAGE_IDLE_RETRIES) + monkeypatch.delenv(spend.JOURNAL_ALIAS, raising=False) + monkeypatch.delenv("ALK_READING_MAX_DURATION_SECONDS", raising=False) + monkeypatch.setenv("ALK_HARNESS_MODEL", "gemini-2.5-flash") + # The machine's own secrets files must not decide what a test sees masked. + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (tmp_path / "no-secrets.json",)) + monkeypatch.setattr(reading_entry, "_secret_values", None) + # A scripted model that never answers must end a test, not hang it. + monkeypatch.setattr(agent_reading, "_soft_limit", lambda: 20.0) + monkeypatch.setattr(agent_reading, "_INTERRUPT_WAIT_SECONDS", 1.0) + spend._stages.clear() + spend.journal_to(None) + yield + spend._stages.clear() + spend.journal_to(None) + + +# -- the fake provider -------------------------------------------------------------------------- + + +class _Live: + """One session of the fake backend; each message sent plays the next scripted turn.""" + + def __init__(self, backend: "_Backend", spec) -> None: + self.backend = backend + self.spec = spec + self.interrupted = asyncio.Event() + + async def start(self) -> None: + if self.backend.start_raises: + raise FileNotFoundError("the Claude CLI was not found") + + async def stop(self) -> None: + output = self.backend.output + self.backend.stopped.append( + { + "failure": (output / "failure.json").exists(), + "agent": (output / "agent.json").exists(), + } + ) + + async def send(self, message: str) -> None: + self.backend.sent.append(message) + + async def interrupt(self) -> bool: + self.backend.interrupts.append(time.monotonic()) + if self.backend.interrupt_raises: + raise RuntimeError("the interrupt control request timed out") + self.interrupted.set() + return True + + def replies(self): + turn = self.backend.turns.pop(0) if self.backend.turns else _ends() + return turn(self) + + +class _Backend: + default_model = "gemini-2.5-flash" + + def __init__(self, name: str, output: Path, turns: list) -> None: + self.name = name + self.output = output + self.turns = turns + self.created: list = [] + self.sent: list[str] = [] + self.interrupts: list[float] = [] + self.stopped: list[dict] = [] + self.seen: list = [] + self.interrupt_raises = False + self.start_raises = False + self.submissions = 0 + + def create(self, spec) -> _Live: + self.created.append(spec) + return _Live(self, spec) + + def can_drive(self, model: str) -> bool: + return True + + +@pytest.fixture +def model(tmp_path, monkeypatch): + def install(*turns, name: str = "claude", output: Path | None = None) -> _Backend: + fake = _Backend(name, output or tmp_path / "reading", list(turns)) + monkeypatch.setattr(session, "resolve", lambda *a, **k: fake) + return fake + + return install + + +def _done(**fields) -> StageDone: + return StageDone(**{"outcome": "success", "turns": 1, "cost_usd": 0.0, **fields}) + + +def _ends(**fields): + async def turn(live): + yield _done(**fields) + + return turn + + +async def _gate(live: _Live, arguments: dict) -> None: + """What the model's call does: the real submit_contract from the reading's own spec.""" + tool = next( + spec for spec in live.spec.servers[CONTRACT_SERVER].tools if spec.name == "submit_contract" + ) + output = live.backend.output + # A first thin submission is sent back once, so the model submits again. + for _ in range(3): + live.backend.submissions += 1 + await tool.handler(dict(arguments)) + if (output / "contract.json").exists() or (output / "failure.json").exists(): + return + + +def _submits(arguments: dict, **fields): + async def turn(live): + await _gate(live, arguments) + yield _done(**fields) + + return turn + + +def _save(output: Path, contract: dict | None = None, *, text: str | None = None) -> None: + output.mkdir(parents=True, exist_ok=True) + (output / "contract.json").write_text( + text if text is not None else json.dumps(contract), encoding="utf-8" + ) + + +def _writes(contract: dict | None = None, *, text: str | None = None, **fields): + """A turn whose contract the gate has already accepted and saved.""" + + async def turn(live): + _save(live.backend.output, contract, text=text) + yield _done(**fields) + + return turn + + +async def _quiet(live): + await asyncio.Event().wait() + yield _done() + + +# -- the job, the source and the run ------------------------------------------------------------ + + +def _job(*, subfolder: str | None = None, source: dict | None = None, metadata=None) -> dict: + job = { + "job_id": "job", + "run_id": "run", + "execution": "hosted", + "source": source + or { + "kind": "github", + "visibility": "public", + "repository": "acme/agent", + "commit_sha": "a" * 40, + **({"subfolder": subfolder} if subfolder else {}), + }, + "runtime": {"isolation": "dedicated_vm"}, + "agent": {"connector": "auto"}, + } + if metadata is not None: + job["metadata"] = metadata + return job + + +def _repo(root: Path) -> Path: + source = root / "source" + (source / SUPPORT).mkdir(parents=True) + (source / SUPPORT / "agent.py").write_text("x = 1\n", encoding="utf-8") + (source / SCRAPER).mkdir(parents=True) + (source / SCRAPER / "agent.py").write_text( + "from playwright.sync_api import sync_playwright\nsync_playwright()\n", encoding="utf-8" + ) + return source + + +def _run(root: Path, job: dict | None = None, *, source: Path | None = None, output=None) -> int: + job_path = root / "job.json" + job_path.write_text(json.dumps(job or _job()), encoding="utf-8") + return run_reading_mode( + agent_reading.READ_AGENT, + job_path=job_path, + source=source if source is not None else root / "source", + output=output if output is not None else root / "reading", + ) + + +def _failure_in(output: Path) -> dict: + return json.loads((output / "failure.json").read_text(encoding="utf-8")) + + +def _cost_in(output: Path) -> dict: + return json.loads((output / "cost.json").read_text(encoding="utf-8")) + + +def _names(output: Path) -> set[str]: + return {path.name for path in output.iterdir()} + + +def _assert_cost(output: Path, total: float, complete: bool) -> None: + cost = _cost_in(output) + assert cost["total_usd"] == pytest.approx(total) + assert cost["complete"] is complete + + +def _short_idle(monkeypatch, seconds: float = 0.2) -> None: + real = agent_reading.open_stage + + def opened(*args, **kwargs): + stage, destination = real(*args, **kwargs) + stage.spec.idle_timeout_seconds = seconds + return stage, destination + + monkeypatch.setattr(agent_reading, "open_stage", opened) + + +def _short_limit(monkeypatch, soft: float = 0.3, wait: float = 0.3) -> None: + monkeypatch.setattr(agent_reading, "_soft_limit", lambda: soft) + monkeypatch.setattr(agent_reading, "_INTERRUPT_WAIT_SECONDS", wait) + + +# -- R1 to R9: the files a reading leaves ------------------------------------------------------- + + +def test_a_reading_leaves_the_contract_the_document_and_the_cost(tmp_path, model): + source = _repo(tmp_path) + model(_submits(CHAT, cost_usd=0.25)) + output = tmp_path / "reading" + + assert _run(tmp_path, _job(subfolder=SUPPORT)) == 0 + + assert _names(output) == {"contract.json", "agent.json", "cost.json"} + document = json.loads((output / "agent.json").read_text(encoding="utf-8")) + assert set(document) == {"sdk_version", "agent"} + assert document["sdk_version"] == fi.alk.__version__ + agent = document["agent"] + assert agent["extension"]["source"] == {"subfolder": SUPPORT} + contract = AgentContract.model_validate( + json.loads((output / "contract.json").read_text(encoding="utf-8")) + ).model_dump(mode="json") + framework = detect_framework(source / SUPPORT, repo_root=source, package_dir=None) + assert agent == build_authored_agent(contract, subfolder=SUPPORT, framework=framework) + _assert_cost(output, 0.25, True) + + +@pytest.mark.parametrize( + ("write", "expected"), + [ + ( + lambda folder: (folder / "graph.py").write_text( + "from langgraph.graph import StateGraph\nStateGraph(int)\n", encoding="utf-8" + ), + {"name": "langgraph", "version": None}, + ), + ( + lambda folder: shutil.copy(EXAMPLES / "llamaindex_agent.py", folder), + {"name": "other", "version": None, "label": "llama_index"}, + ), + ], + ids=["langgraph", "other"], +) +def test_the_framework_is_a_builds_shape(tmp_path, model, write, expected): + source = _repo(tmp_path) + write(source / SUPPORT) + model(_writes(IN_SUPPORT)) + + assert _run(tmp_path, _job(subfolder=SUPPORT)) == 0 + + agent = json.loads((tmp_path / "reading" / "agent.json").read_text(encoding="utf-8")) + assert agent["agent"]["extension"]["framework"] == expected + + +def test_with_no_folder_named_the_framework_comes_from_the_contracts_folder(tmp_path, model): + source = _repo(tmp_path) + shutil.copy(EXAMPLES / "llamaindex_agent.py", source / SUPPORT) + # A different framework at the root, so reading the root instead would show. + (source / "graph.py").write_text( + "from langgraph.graph import StateGraph\nStateGraph(int)\n", encoding="utf-8" + ) + model(_writes({**CHAT, "runtime": {"language": "python", "workdir": SUPPORT}})) + + assert _run(tmp_path) == 0 + + agent = json.loads((tmp_path / "reading" / "agent.json").read_text(encoding="utf-8")) + assert agent["agent"]["extension"]["framework"] == { + "name": "other", + "version": None, + "label": "llama_index", + } + + +def test_a_browser_agent_is_refused_by_the_real_gate(tmp_path, model): + _repo(tmp_path) + fake = model(_submits({"modality": "browser"}, cost_usd=0.37)) + output = tmp_path / "reading" + + assert _run(tmp_path, _job(subfolder=SCRAPER)) == 78 + + assert _failure_in(output)["code"] == "agent_type_unsupported" + assert fake.submissions == 1 + assert len(fake.sent) == 1 + assert not (output / "agent.json").exists() + assert not (output / "contract.json").exists() + _assert_cost(output, 0.37, True) + + +def test_a_missing_folder_is_refused_as_a_build_refuses_it(tmp_path, model): + _repo(tmp_path) + fake = model() + output = tmp_path / "reading" + + assert _run(tmp_path, _job(subfolder="agents/missing")) == 78 + + failure = _failure_in(output) + assert (failure["code"], failure["stage"]) == ("subfolder_not_found", "acquiring_source") + assert fake.created == [] + _assert_cost(output, 0.0, True) + + +def _no_source(root: Path) -> Path: + return root / "source" + + +def _a_file(root: Path) -> Path: + (root / "source").write_text("not a folder", encoding="utf-8") + return root / "source" + + +def _an_empty_folder(root: Path) -> Path: + (root / "source").mkdir() + return root / "source" + + +@pytest.mark.parametrize( + ("make", "subfolder"), + [ + (_no_source, None), + (_a_file, None), + (_an_empty_folder, None), + (_no_source, SUPPORT), + ], + ids=["missing", "a_file", "empty", "missing_with_a_folder"], +) +def test_a_missing_or_empty_source_is_retryable_and_costs_nothing( + tmp_path, model, make, subfolder +): + source = make(tmp_path) + fake = model() + output = tmp_path / "reading" + + assert _run(tmp_path, _job(subfolder=subfolder), source=source) == 1 + + failure = _failure_in(output) + assert (failure["code"], failure["domain"], failure["retryable"]) == ( + "reading_source_empty", + "platform_sync", + True, + ) + assert fake.created == [] + _assert_cost(output, 0.0, True) + + +@pytest.mark.parametrize( + "kind", + [SourceKind.REMOTE, SourceKind.PROVIDER], + ids=["remote", "provider"], +) +def test_only_github_or_an_upload_is_read(tmp_path, model, kind): + _repo(tmp_path) + fake = model() + parsed = HarnessJob.model_validate(_job()) + + other = parsed.model_copy( + update={"source": parsed.source.model_copy(update={"kind": kind})} + ) + refused = agent_reading._check_job(other) + assert refused is not None + assert (refused.code, refused.domain.value, refused.retryable) == ( + READING_JOB_INVALID, + "platform_sync", + False, + ) + for accepted in (SourceKind.GITHUB, SourceKind.ARCHIVE): + assert ( + agent_reading._check_job( + parsed.model_copy( + update={"source": parsed.source.model_copy(update={"kind": accepted})} + ) + ) + is None + ) + + if kind is SourceKind.REMOTE: + remote = _job(source={"kind": "remote", "endpoint": "https://agent.example.com/chat"}) + HarnessJob.model_validate(remote) + assert _run(tmp_path, remote) == 78 + assert _failure_in(tmp_path / "reading")["code"] == READING_JOB_INVALID + assert fake.created == [] + + +def test_a_folder_mismatch_is_an_invalid_document(tmp_path, model): + _repo(tmp_path) + # Saved with no folder recorded, for a job that names one. + model(_writes(CHAT, cost_usd=0.11)) + output = tmp_path / "reading" + + assert _run(tmp_path, _job(subfolder=SUPPORT)) == 78 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_document_invalid", False) + assert failure["details"]["code"] == "subfolder_mismatch" + assert (output / "contract.json").exists() + assert not (output / "agent.json").exists() + _assert_cost(output, 0.11, True) + + +def _stored_size(inner: dict) -> int: + return len(json.dumps(inner, ensure_ascii=False).encode("utf-8")) + + +def _body_size(inner: dict) -> int: + return len( + json.dumps( + {"sdk_version": fi.alk.__version__, "agent": inner}, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + ) + + +def _sized(target: int, measure, items: int) -> dict: + """An inner document that ``measure`` puts at exactly ``target`` bytes.""" + + def inner(pad: int) -> dict: + return {"items": [0] * items, "pad": "a" * pad} + + return inner(target - measure(inner(0))) + + +@pytest.mark.parametrize( + ("inner", "passes"), + [ + # Many separators: over by the platform's stored measure, under by the file's. + (lambda: _sized(MIB + 1, _stored_size, 300_000), False), + (lambda: _sized(MIB, _stored_size, 300_000), True), + (lambda: _sized(MIB + 1, _body_size, 0), False), + ], + ids=["stored_over", "stored_exactly", "file_over"], +) +def test_an_oversized_document_is_refused(tmp_path, model, monkeypatch, inner, passes): + _repo(tmp_path) + document = inner() + if not passes and document["items"]: + assert _body_size(document) <= MIB + monkeypatch.setattr(agent_reading, "build_authored_agent", lambda *a, **k: document) + model(_writes(CHAT, cost_usd=0.05)) + output = tmp_path / "reading" + + code = _run(tmp_path) + + if passes: + assert code == 0 + assert (output / "agent.json").stat().st_size == _body_size(document) + else: + assert code == 78 + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_too_large", False) + assert not (output / "agent.json").exists() + assert (output / "contract.json").exists() + _assert_cost(output, 0.05, True) + + +@pytest.mark.parametrize(("size", "passes"), [(5 * MIB + 1, False), (5 * MIB, True)]) +def test_an_oversized_contract_is_refused(tmp_path, model, size, passes): + _repo(tmp_path) + text = json.dumps(CHAT) + # Trailing spaces still parse, so only the file's size changes. + model(_writes(text=text + " " * (size - len(text.encode("utf-8"))), cost_usd=0.05)) + output = tmp_path / "reading" + + code = _run(tmp_path) + + assert (output / "contract.json").stat().st_size == size + if passes: + assert code == 0 + assert (output / "agent.json").exists() + else: + assert code == 78 + assert _failure_in(output)["code"] == "agent_reading_too_large" + assert not (output / "agent.json").exists() + _assert_cost(output, 0.05, True) + + +# -- R10 to R19: how a turn's ending is read ---------------------------------------------------- + + +def test_no_contract_after_the_reminder_is_incomplete(tmp_path, model): + _repo(tmp_path) + # Claude's results carry the session's running total. + fake = model(_ends(cost_usd=0.07875), _ends(cost_usd=0.1575)) + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + assert len(fake.sent) == 2 + assert fake.sent[1] == agent_reading._REMINDER + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_incomplete", False) + assert failure["details"] == {"outcome": "success", "turns": 2} + _assert_cost(output, 0.1575, True) + + +# Gemini's call limit arrives as a raise, so its own test below drives the real backend; a +# stream that ends with no final answer is the outcome here. One turn, well under the budget, +# so only the outcome can mark it. +_TURN_LIMIT_SHAPES = [ + ("claude", {"outcome": "error_max_turns", "is_error": True}), + ("vertex-gemini", {"outcome": "max_turns", "is_error": True, "turns": 1}), +] + + +@pytest.mark.parametrize(("name", "ending"), _TURN_LIMIT_SHAPES, ids=["claude", "gemini"]) +def test_the_turn_limit_is_incomplete_not_a_model_error(tmp_path, model, name, ending): + _repo(tmp_path) + fake = model(_ends(cost_usd=0.2, **ending), name=name) + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + assert len(fake.sent) == 1 + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_incomplete", False) + assert failure["details"]["outcome"] == ending["outcome"] + _assert_cost(output, 0.2, True) + + +@pytest.mark.parametrize("status", [408, 409, 429, 500, 529]) +def test_a_provider_error_is_retryable(tmp_path, model, status): + _repo(tmp_path) + model(_ends(cost_usd=0.3, is_error=True, api_error_status=status, errors=["overloaded"])) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_model_error", True) + assert failure["details"] == {"status": status} + assert "overloaded" in failure["message"] + _assert_cost(output, 0.3, True) + + +@pytest.mark.parametrize( + ("name", "ending"), + [ + ("claude", {"outcome": "success", "is_error": True}), + # A provider exception, well inside the call budget. + ("vertex-gemini", {"outcome": "failed", "is_error": True, "turns": 1}), + ], + ids=["claude", "gemini"], +) +def test_a_failure_with_no_status_is_a_retryable_model_error(tmp_path, model, name, ending): + _repo(tmp_path) + fake = model(_ends(**ending), _ends(**ending), name=name) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + assert len(fake.sent) == 2 + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_model_error", True) + assert failure["details"] == {"status": None} + # A request cut with no status may be billed without a total reaching the stage. + assert _cost_in(output)["complete"] is False + + +@pytest.mark.parametrize("status", [400, 401, 404]) +def test_a_refusing_4xx_is_not_retried(tmp_path, model, status): + _repo(tmp_path) + model(_ends(cost_usd=0.01, is_error=True, api_error_status=status)) + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_model_error", False) + assert failure["details"] == {"status": status} + + +def test_a_gemini_grpc_code_is_retryable(tmp_path, model): + _repo(tmp_path) + model( + _ends(outcome="failed", is_error=True, api_error_status=14), name="vertex-gemini" + ) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_model_error", True) + assert failure["details"] == {"status": 14} + + +def test_a_failed_opening_gets_no_reminder(tmp_path, model): + _repo(tmp_path) + fake = model(_ends(is_error=True, api_error_status=529)) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + assert len(fake.sent) == 1 + assert _failure_in(output)["code"] == "agent_reading_model_error" + + +@pytest.mark.parametrize(("name", "ending"), _TURN_LIMIT_SHAPES, ids=["claude", "gemini"]) +def test_a_turn_limit_opening_gets_no_reminder(tmp_path, model, name, ending): + _repo(tmp_path) + fake = model(_ends(**ending), name=name) + + _run(tmp_path) + + assert len(fake.sent) == 1 + + +def test_an_error_flag_without_a_status_still_gets_the_reminder(tmp_path, model): + _repo(tmp_path) + fake = model( + _ends(outcome="success", is_error=True, cost_usd=0.1), + _writes(CHAT, cost_usd=0.2), + ) + output = tmp_path / "reading" + + assert _run(tmp_path) == 0 + + assert fake.sent[1:] == [agent_reading._REMINDER] + assert (output / "agent.json").exists() + # The first turn's error had no status, so its spend may not all be in the total. + _assert_cost(output, 0.2, False) + + +# Priced, so the stop's cost is checked against the table and not left at zero. +_PRICED_GEMINI = "gemini-3.7-flash" + + +def _real_gemini(monkeypatch, reply) -> tuple[list[str], list[str]]: + """The real Gemini backend and ADK runner; only the model is an in-memory one. + + ``reply(who, calls)`` gives the reply to the agent named ``who``; ``calls`` names every + agent that has called so far. Returns the messages sent and the calls made. + """ + from google.adk.models.base_llm import BaseLlm + + from fi.alk.harness.backends import vertex_gemini + + calls: list[str] = [] + + class Scripted(BaseLlm): + async def generate_content_async(self, llm_request, stream=False): + labels = (llm_request.config.labels or {}) if llm_request.config else {} + calls.append(labels.get("adk_agent_name", "")) + yield reply(calls[-1], calls) + + # The session writes these into the environment; set here first, they are put back after. + monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "test-project") + monkeypatch.setenv("GOOGLE_GENAI_USE_VERTEXAI", "TRUE") + monkeypatch.setenv("GOOGLE_CLOUD_LOCATION", "global") + monkeypatch.setenv("ALK_HARNESS_MODEL", _PRICED_GEMINI) + monkeypatch.setattr(vertex_gemini, "_llm", lambda name: Scripted(model=name)) + backend = vertex_gemini.VertexGeminiBackend() + monkeypatch.setattr(session, "resolve", lambda *a, **k: backend) + sent: list[str] = [] + real_send = vertex_gemini.VertexGeminiSession.send + + async def send(self, message): + sent.append(message) + return await real_send(self, message) + + monkeypatch.setattr(vertex_gemini.VertexGeminiSession, "send", send) + return sent, calls + + +def _gemini_usage(): + from google.genai import types + + return types.GenerateContentResponseUsageMetadata( + prompt_token_count=1000, candidates_token_count=10 + ) + + +def _gemini_calls(name: str, args: dict, n: int): + from google.adk.models.llm_response import LlmResponse + from google.genai import types + + call = types.FunctionCall(name=name, args=args, id=f"call-{n}") + return LlmResponse( + content=types.Content(role="model", parts=[types.Part(function_call=call)]), + usage_metadata=_gemini_usage(), + ) + + +def _gemini_priced(calls: int) -> float: + # The fake's 1,000 tokens in and 10 out per call, at the table's own rates, kept to the + # six places the spend record keeps. + from fi.alk.harness.backends.vertex_gemini import PRICES_PER_MILLION + + rate_in, rate_out = PRICES_PER_MILLION[_PRICED_GEMINI][:2] + return round(calls * (1000 * rate_in + 10 * rate_out) / 1_000_000, 6) + + +def test_a_gemini_call_limit_is_incomplete_with_no_reminder(tmp_path, model, monkeypatch): + sent, calls = _real_gemini( + monkeypatch, lambda who, calls: _gemini_calls("Glob", {"pattern": "*.py"}, len(calls)) + ) + monkeypatch.setattr(agent_reading, "BUILDER_TURNS", 3) + _repo(tmp_path) + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + assert len(sent) == 1 + assert len(calls) == 3 + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_incomplete", False) + # The backend's own count reaches the budget when ADK stops the calls. + assert failure["details"] == {"outcome": "failed", "turns": 3} + # ADK stops before the call it refuses, so every call made is in the total. + _assert_cost(output, _gemini_priced(3), True) + + +def test_a_gemini_call_limit_after_a_malformed_worker_reply_gets_no_reminder( + tmp_path, model, monkeypatch +): + from google.adk.models.llm_response import LlmResponse + + def reply(who, calls): + if who != "agent_reader": + return _gemini_calls("agent_reader", {"request": "Trace the add tool."}, len(calls)) + if calls.count("agent_reader") == 1: + # Gemini's malformed call: no content, so the backend doesn't count it as a turn, + # but ADK still spends a call on it. + return LlmResponse( + error_code="MALFORMED_FUNCTION_CALL", + error_message="Malformed function call", + usage_metadata=_gemini_usage(), + ) + return _gemini_calls("Glob", {"pattern": "*.py"}, len(calls)) + + sent, calls = _real_gemini(monkeypatch, reply) + monkeypatch.setattr(agent_reading, "BUILDER_TURNS", 6) + _repo(tmp_path) + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + # One budget, not two: no reminder opened a second one. + assert len(sent) == 1 + assert len(calls) == 6 + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_incomplete", False) + # The backend's count falls short of the budget; ADK's own words mark the stop. + assert failure["details"]["turns"] < 6 + _assert_cost(output, _gemini_priced(6), True) + + +def test_an_earlier_failed_turn_is_still_a_model_error(tmp_path, model): + _repo(tmp_path) + fake = model(_ends(outcome="success", is_error=True), _ends()) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + assert len(fake.sent) == 2 + assert _failure_in(output)["code"] == "agent_reading_model_error" + + +def test_a_cut_on_the_last_allowed_call_is_not_a_turn_limit(tmp_path, model): + _repo(tmp_path) + fake = model( + _ends(outcome="failed", is_error=True, turns=BUILDER_TURNS, errors=["connection reset"]), + _ends(), + name="vertex-gemini", + ) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + assert fake.sent[1:] == [agent_reading._REMINDER] + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_model_error", True) + # The cut request may be billed without its total reaching the stage. + assert _cost_in(output)["complete"] is False + + +@pytest.mark.parametrize( + "said", + [ + "Tool 'Max number of llm calls limit of `6` exceeded' not found", + "invalid literal for int() with base 10: 'Max number of llm calls limit of `6` exceeded'", + "Max number of llm calls limit of `6` exceeded, said the model", + ], + ids=["tool-name", "offset", "trailing"], +) +def test_the_call_limit_phrase_inside_other_text_is_a_model_error(tmp_path, model, said): + _repo(tmp_path) + fake = model( + _ends(outcome="failed", is_error=True, errors=[said]), _ends(), name="vertex-gemini" + ) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + assert len(fake.sent) == 2 + assert _failure_in(output)["code"] == "agent_reading_model_error" + + +@pytest.mark.parametrize( + ("name", "cut", "limit"), + [ + ("claude", {"outcome": "success", "is_error": True}, _TURN_LIMIT_SHAPES[0][1]), + ( + "vertex-gemini", + {"outcome": "failed", "is_error": True}, + _TURN_LIMIT_SHAPES[1][1], + ), + ( + "vertex-gemini", + {"outcome": "failed", "is_error": True}, + { + "outcome": "failed", + "is_error": True, + "errors": ["Max number of llm calls limit of `200` exceeded"], + }, + ), + ], + ids=["claude", "gemini", "gemini-call-limit"], +) +def test_a_turn_limit_on_the_reminder_wins_over_an_earlier_cut( + tmp_path, model, name, cut, limit +): + _repo(tmp_path) + fake = model(_ends(**cut), _ends(**limit), name=name) + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + assert len(fake.sent) == 2 + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_incomplete", False) + # The opening's cut may still be billed without its total reaching the stage. + assert _cost_in(output)["complete"] is False + + +def test_a_cut_on_the_reminder_leaves_the_cost_incomplete(tmp_path, model): + _repo(tmp_path) + fake = model(_ends(cost_usd=0.07875), _ends(outcome="success", is_error=True)) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + assert len(fake.sent) == 2 + assert _failure_in(output)["code"] == "agent_reading_model_error" + assert _cost_in(output)["complete"] is False + + +# -- R20 to R27: a quiet stream, the soft limit, and what was saved ------------------------------ + + +def test_an_idle_stream_is_not_replayed(tmp_path, model, monkeypatch): + _repo(tmp_path) + monkeypatch.setattr(session, "STAGE_IDLE_RETRIES", 1) + _short_idle(monkeypatch) + fake = model(_quiet) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + assert len(fake.created) == 1 + assert fake.interrupts == [] + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_model_error", True) + _assert_cost(output, 0.0, False) + + +def test_a_contract_saved_before_an_idle_stream_is_kept(tmp_path, model, monkeypatch): + _repo(tmp_path) + _short_idle(monkeypatch) + + async def saves_then_goes_quiet(live): + _save(live.backend.output, CHAT) + await asyncio.Event().wait() + yield _done() + + model(saves_then_goes_quiet) + output = tmp_path / "reading" + + assert _run(tmp_path) == 0 + + assert (output / "agent.json").exists() + assert not (output / "failure.json").exists() + _assert_cost(output, 0.0, False) + + +def test_the_soft_limit_interrupts_once_and_charges_what_arrives(tmp_path, model, monkeypatch): + _repo(tmp_path) + _short_limit(monkeypatch, wait=2.0) + + async def answers_the_interrupt(live): + await live.interrupted.wait() + yield _done(outcome="error_during_execution", is_error=True, cost_usd=0.42) + + fake = model(answers_the_interrupt) + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + assert len(fake.sent) == 1 + assert len(fake.interrupts) == 1 + assert _failure_in(output)["code"] == "agent_reading_timed_out" + _assert_cost(output, 0.42, False) + + +def test_the_soft_limit_marks_cost_incomplete_when_nothing_arrives(tmp_path, model, monkeypatch): + _repo(tmp_path) + _short_limit(monkeypatch) + fake = model(_quiet) + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + assert len(fake.interrupts) == 1 + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_timed_out", False) + _assert_cost(output, 0.0, False) + + +def test_an_interrupt_that_raises_is_still_a_timeout(tmp_path, model, monkeypatch): + _repo(tmp_path) + _short_limit(monkeypatch) + fake = model(_quiet) + fake.interrupt_raises = True + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + assert _failure_in(output)["code"] == "agent_reading_timed_out" + + +def test_a_contract_saved_at_the_soft_limit_is_kept(tmp_path, model, monkeypatch): + _repo(tmp_path) + _short_limit(monkeypatch, wait=2.0) + + async def saves_when_interrupted(live): + await live.interrupted.wait() + _save(live.backend.output, CHAT) + yield _done(outcome="error_during_execution", is_error=True, cost_usd=0.3) + + model(saves_when_interrupted) + output = tmp_path / "reading" + + assert _run(tmp_path) == 0 + + assert (output / "agent.json").exists() + assert not (output / "failure.json").exists() + _assert_cost(output, 0.3, False) + + +def test_nothing_is_saved_after_the_soft_limit_decides(tmp_path, model, monkeypatch): + _repo(tmp_path) + _short_limit(monkeypatch, soft=0.3, wait=0.3) + + async def saves_too_late(live): + # Ignores the interrupt, and would save only after the wait has run out. + await asyncio.sleep(1.5) + await _gate(live, CHAT) + yield _done() + + fake = model(saves_too_late) + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + # The session was closed before either result file could be written. + assert fake.stopped == [{"failure": False, "agent": False}] + assert not (output / "contract.json").exists() + assert _failure_in(output)["code"] == "agent_reading_timed_out" + + +def test_a_saved_contract_wins_over_an_error_flag(tmp_path, model): + _repo(tmp_path) + model(_writes(CHAT, is_error=True, cost_usd=0.1)) + output = tmp_path / "reading" + + assert _run(tmp_path) == 0 + + assert (output / "agent.json").exists() + assert not (output / "failure.json").exists() + + +def test_a_turn_with_no_ending_leaves_the_cost_incomplete(tmp_path, model): + _repo(tmp_path) + + async def saves_and_never_reports(live): + _save(live.backend.output, CHAT) + for nothing in (): + yield nothing + + fake = model(saves_and_never_reports) + output = tmp_path / "reading" + + assert _run(tmp_path) == 0 + + assert len(fake.sent) == 1 + assert (output / "agent.json").exists() + _assert_cost(output, 0.0, False) + + +# -- R28 to R32: what is charged ---------------------------------------------------------------- + + +def test_the_cost_follows_each_turn(tmp_path, model): + _repo(tmp_path) + + async def reads_the_cost(live): + cost = _cost_in(live.backend.output) + live.backend.seen.append((cost["total_usd"], cost["complete"])) + yield _done(cost_usd=0.1575) + + fake = model(_ends(cost_usd=0.07875), reads_the_cost) + + _run(tmp_path) + + assert fake.seen == [(pytest.approx(0.07875), False)] + + +def _stage_entry() -> dict: + [entry] = spend.snapshot()["stages"] + return entry + + +def test_claude_running_totals_are_counted_once(tmp_path, model): + _repo(tmp_path) + model( + _ends(cost_usd=0.07875, tokens_in=100_000), + _ends(cost_usd=0.1575, tokens_in=200_000), + ) + output = tmp_path / "reading" + + _run(tmp_path) + + _assert_cost(output, 0.1575, True) + assert _stage_entry()["tokens_in"] == 200_000 + + +def test_gemini_counts_are_added_as_reported(tmp_path, model): + _repo(tmp_path) + model( + _ends(cost_usd=0.1, tokens_in=1_000), + _ends(cost_usd=0.1, tokens_in=1_000), + name="vertex-gemini", + ) + output = tmp_path / "reading" + + _run(tmp_path) + + _assert_cost(output, 0.2, True) + assert _stage_entry()["tokens_in"] == 2_000 + + +class _MessagesStub(BaseHTTPRequestHandler): + """The Messages API on 127.0.0.1, answering every request with fixed usage.""" + + seen: list[str] = [] + + def log_message(self, *args) -> None: + pass + + def _json(self, body: dict) -> None: + raw = json.dumps(body).encode() + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + + def do_HEAD(self) -> None: + self.send_response(200) + self.end_headers() + + def do_GET(self) -> None: + self._json({"data": [], "has_more": False}) + + def do_POST(self) -> None: + raw = self.rfile.read(int(self.headers.get("content-length") or 0)) + try: + request = json.loads(raw) + except ValueError: + request = {} + self.seen.append(self.path) + if "count_tokens" in self.path: + self._json({"input_tokens": 10}) + return + model_name = request.get("model", "claude-sonnet-4-5") + if not request.get("stream"): + self._json( + { + "id": "msg_x", + "type": "message", + "role": "assistant", + "model": model_name, + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 1}, + } + ) + return + self.send_response(200) + self.send_header("content-type", "text/event-stream") + self.send_header("cache-control", "no-cache") + self.end_headers() + + def event(name: str, data: dict) -> None: + self.wfile.write(f"event: {name}\ndata: {json.dumps(data)}\n\n".encode()) + self.wfile.flush() + + event( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": model_name, + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": { + "input_tokens": 100_000, + "output_tokens": 1, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + }, + }, + ) + event( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ) + event( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "done"}}, + ) + event("content_block_stop", {"type": "content_block_stop", "index": 0}) + event( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 1000}, + }, + ) + event("message_stop", {"type": "message_stop"}) + + +def test_the_claude_cli_reports_running_totals(tmp_path): + import claude_agent_sdk + from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient, ResultMessage + + cli = Path(claude_agent_sdk.__file__).parent / "_bundled" / "claude" + if not cli.is_file(): + pytest.fail(f"the Claude CLI bundled with claude-agent-sdk was not found at {cli}") + _MessagesStub.seen = [] + server = ThreadingHTTPServer(("127.0.0.1", 0), _MessagesStub) + threading.Thread(target=server.serve_forever, daemon=True).start() + nowhere = "http://127.0.0.1:9" + env = { + "ANTHROPIC_BASE_URL": f"http://127.0.0.1:{server.server_address[1]}", + "ANTHROPIC_API_KEY": "placeholder-not-a-key", + "ANTHROPIC_AUTH_TOKEN": "", + # Another test in the same process may leave these set, and the CLI would go there. + "CLAUDE_CODE_USE_VERTEX": "", + "CLAUDE_CODE_USE_BEDROCK": "", + "CLAUDE_CODE_USE_FOUNDRY": "", + "ANTHROPIC_VERTEX_PROJECT_ID": "", + "ANTHROPIC_VERTEX_BASE_URL": nowhere, + "ANTHROPIC_BEDROCK_BASE_URL": nowhere, + "ANTHROPIC_FOUNDRY_BASE_URL": nowhere, + "OTEL_EXPORTER_OTLP_ENDPOINT": nowhere, + "CLAUDE_CODE_ENABLE_TELEMETRY": "0", + "DISABLE_TELEMETRY": "1", + "DISABLE_ERROR_REPORTING": "1", + "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1", + "DISABLE_AUTOUPDATER": "1", + "CLAUDE_CONFIG_DIR": str(tmp_path / "config"), + } + work = tmp_path / "work" + work.mkdir() + options = ClaudeAgentOptions( + model="claude-sonnet-4-5", + max_turns=5, + env=env, + cwd=str(work), + allowed_tools=[], + setting_sources=[], + cli_path=str(cli), + ) + results: list = [] + + async def two_queries() -> None: + async with ClaudeSDKClient(options=options) as client: + for prompt in ("first", "second, as the reminder would be"): + await client.query(prompt) + async for message in client.receive_response(): + if isinstance(message, ResultMessage): + results.append(message) + + try: + asyncio.run(asyncio.wait_for(two_queries(), 120)) + finally: + server.shutdown() + server.server_close() + + asked = [path for path in _MessagesStub.seen if "count_tokens" not in path] + assert len(asked) == 2, asked + first, second = results + assert (first.is_error, second.is_error) == (False, False) + assert first.total_cost_usd > 0 + assert second.total_cost_usd == pytest.approx(2 * first.total_cost_usd) + + def input_tokens(result) -> int: + return sum( + int((usage or {}).get("inputTokens", 0)) + for usage in (result.model_usage or {}).values() + ) + + assert [input_tokens(first), input_tokens(second)] == [100_000, 200_000] + + +def test_a_forbidden_model_is_not_retried(tmp_path, model): + _repo(tmp_path) + model(_ends(cost_usd=0.2, models={"claude-sonnet-4-5"})) + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_model_error", False) + # Spent before the stop, and that turn never finished being kept. + _assert_cost(output, 0.2, False) + + +# -- R33 to R36: the stage the reading runs ----------------------------------------------------- + + +_EVALS = [{"name": "tone_check", "description": "Checks the tone.", "modality": "chat"}] + + +def test_the_reading_uses_the_builders_settings(tmp_path, model): + _repo(tmp_path) + fake = model() + + _run(tmp_path, _job(metadata={"available_evals": _EVALS})) + + [spec] = fake.created + assert spec.max_turns == BUILDER_TURNS == 200 + assert spec.model == chosen_model() + assert spec.thinking is False + assert "tone_check" in spec.system_prompt + assert CONTRACT_SERVER in spec.servers + + +def test_the_claude_backend_accepts_the_reading_spec(tmp_path, model, monkeypatch): + from fi.alk.harness.backends.claude import ClaudeBackend + + monkeypatch.setenv("ALK_CLAUDE_GATEWAY_URL", "http://127.0.0.1:9") + monkeypatch.setenv("ALK_CLAUDE_GATEWAY_API_KEY", "placeholder-not-a-key") + _repo(tmp_path) + fake = model() + _run(tmp_path) + [spec] = fake.created + + options = ClaudeBackend().create(spec)._options + try: + assert options.max_turns == 200 + assert getattr(options, "thinking", None) is None + assert any(name.endswith("submit_contract") for name in options.allowed_tools) + finally: + prompt = options.system_prompt + if isinstance(prompt, dict) and prompt.get("path"): + Path(prompt["path"]).unlink(missing_ok=True) + + +def test_the_reminder_is_the_clis_reminder(): + from fi.alk.harness import cli + + tree = ast.parse(textwrap.dedent(inspect.getsource(cli._understand))) + [nudge] = [ + keyword.value + for node in ast.walk(tree) + if isinstance(node, ast.Call) + for keyword in node.keywords + if keyword.arg == "nudge" + ] + assert isinstance(nudge, ast.Constant) + assert nudge.value == agent_reading._REMINDER + + +def test_the_soft_limit_counts_from_the_readings_start(tmp_path, model, monkeypatch): + _repo(tmp_path) + _short_limit(monkeypatch, soft=3.0, wait=0.2) + real = agent_reading.open_stage + + def slow(*args, **kwargs): + time.sleep(2.0) + return real(*args, **kwargs) + + monkeypatch.setattr(agent_reading, "open_stage", slow) + fake = model(_quiet) + started = time.monotonic() + + assert _run(tmp_path) == 78 + + [interrupted] = fake.interrupts + assert 2.8 <= interrupted - started < 4.0 + + +# -- R37 to R41: the switch, the folders, the limit and the last write -------------------------- + + +def _never(*_args, **_kwargs): + raise AssertionError("a reading must not reach the build's setup") + + +def test_the_switch_runs_only_the_reading(tmp_path, model, monkeypatch): + _repo(tmp_path) + monkeypatch.setattr(authoring_entrypoint, "initialize_hosted_compose", _never) + monkeypatch.setattr(authoring_entrypoint, "_load_provider_import_profile", _never) + model(_writes(CHAT)) + job_path = tmp_path / "job.json" + job_path.write_text(json.dumps(_job()), encoding="utf-8") + output = tmp_path / "reading" + + code = authoring_entrypoint.main( + [str(job_path), "--source", str(tmp_path / "source"), "--output", str(output), "--read-agent-only"] + ) + + assert code == 0 + assert (output / "agent.json").exists() + + +def test_a_bad_job_through_main_is_refused(tmp_path, model): + fake = model() + job_path = tmp_path / "job.json" + job_path.write_text("{not a job", encoding="utf-8") + output = tmp_path / "reading" + + code = authoring_entrypoint.main( + [str(job_path), "--source", str(tmp_path / "source"), "--output", str(output), "--read-agent-only"] + ) + + assert code == 78 + assert _failure_in(output)["code"] == READING_JOB_INVALID + assert fake.created == [] + + +def _everything_under(root: Path) -> dict[str, bytes]: + return { + path.relative_to(root).as_posix(): path.read_bytes() + for path in sorted(root.rglob("*")) + if path.is_file() + } + + +@pytest.mark.parametrize("ends", ["succeeds", "fails"]) +def test_nothing_is_written_outside_the_output(tmp_path, model, ends): + source = _repo(tmp_path) + work = tmp_path / "work" + (work / "authoring").mkdir(parents=True) + (work / "authoring" / "kept.txt").write_text("the build's own folder", encoding="utf-8") + (work / "job.json").write_text(json.dumps(_job(subfolder=SUPPORT)), encoding="utf-8") + output = work / "reading" + # The real gate, so the contract lands wherever the reading's own stage writes it. + model(_submits(CHAT) if ends == "succeeds" else _ends(), output=output) + before = _everything_under(tmp_path) + + code = run_reading_mode( + agent_reading.READ_AGENT, job_path=work / "job.json", source=source, output=output + ) + + assert code == (0 if ends == "succeeds" else 78) + assert (output / "agent.json").exists() is (ends == "succeeds") + after = { + name: body + for name, body in _everything_under(tmp_path).items() + if not name.startswith("work/reading/") + } + assert after == before + + +@pytest.mark.parametrize("value", ["abc", "0", "-5", "", "nan", "inf"]) +def test_a_bad_time_limit_falls_back_to_the_default(monkeypatch, value): + monkeypatch.setenv("ALK_READING_MAX_DURATION_SECONDS", value) + + assert _SOFT_LIMIT() == 1800.0 + + +def test_the_hard_limit_is_a_timeout_that_is_not_retried(): + mode = agent_reading.READ_AGENT + + assert mode.timed_out_code == agent_reading.AGENT_READING_TIMED_OUT == "agent_reading_timed_out" + assert mode.timed_out_retryable is False + assert mode.crashed_code == agent_reading.AGENT_READING_CRASHED == "agent_reading_crashed" + + +def test_a_good_time_limit_is_read(monkeypatch): + monkeypatch.setenv("ALK_READING_MAX_DURATION_SECONDS", "120") + + assert _SOFT_LIMIT() == 120.0 + assert agent_reading.READ_AGENT.time_limit_seconds == 1800.0 + 180.0 + + +def test_a_fault_after_a_saved_contract_still_leaves_the_document( + tmp_path, model, monkeypatch, caplog +): + _repo(tmp_path) + + def unreadable(*_args, **_kwargs): + raise PermissionError("the output folder could not be read") + + monkeypatch.setattr(agent_reading, "_classify", unreadable) + model(_writes(CHAT, cost_usd=0.1)) + output = tmp_path / "reading" + + with caplog.at_level("WARNING", logger=agent_reading.__name__): + assert _run(tmp_path) == 0 + + assert _names(output) == {"contract.json", "agent.json", "cost.json"} + _assert_cost(output, 0.1, False) + # The fault is named by its class only: its text may carry what the customer's code said. + logged = [record.getMessage() for record in caplog.records if record.name == agent_reading.__name__] + assert any("PermissionError" in line for line in logged) + assert not any("could not be read" in line for line in logged) + + +def test_a_session_that_cannot_open_is_a_crash(tmp_path, model): + _repo(tmp_path) + fake = model() + fake.start_raises = True + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_crashed", True) + assert "FileNotFoundError" in failure["message"] + assert fake.sent == [] + _assert_cost(output, 0.0, False) + + +def test_a_failed_document_write_is_not_retried(tmp_path, model, monkeypatch): + _repo(tmp_path) + real = os.replace + + def refuses_the_document(source, destination, *args, **kwargs): + if Path(destination).name == "agent.json": + raise OSError("no space left on the device") + return real(source, destination, *args, **kwargs) + + monkeypatch.setattr(os, "replace", refuses_the_document) + model(_writes(CHAT, cost_usd=0.1)) + output = tmp_path / "reading" + + assert _run(tmp_path) == 78 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_document_invalid", False) + # No document, and no half-written one left beside it. + assert _names(output) == {"contract.json", "cost.json", "failure.json"} + _assert_cost(output, 0.1, True) diff --git a/tests/harness/test_reading_entry.py b/tests/harness/test_reading_entry.py new file mode 100644 index 000000000..484e443fe --- /dev/null +++ b/tests/harness/test_reading_entry.py @@ -0,0 +1,536 @@ +"""The shared way into a reading: what it refuses, the cost it leaves, and its time limit.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from fi.alk.harness import reading_entry, session, spend +from fi.alk.harness.job import FailureDomain, HarnessFailure, HarnessJob, HarnessStage +from fi.alk.harness.reading_entry import ( + MESSAGE_LIMIT, + READING_JOB_INVALID, + READING_OUTPUT_NOT_EMPTY, + ReadingMode, + run_reading_mode, + write_reading_cost, + write_reading_failure, +) + +_DEFAULT_SECRET_FILES = reading_entry._SECRET_FILES + +READ_AGENT_VALUES = { + "name": "read_agent", + "time_limit_seconds": 1980.0, + "timed_out_code": "agent_reading_timed_out", + "timed_out_retryable": False, + "crashed_code": "agent_reading_crashed", +} +FIND_AGENTS_VALUES = { + "name": "find_agents", + "time_limit_seconds": 1800.0, + "timed_out_code": "scan_hard_limit_reached", + "timed_out_retryable": False, + "crashed_code": "scan_crashed", +} + + +@pytest.fixture(autouse=True) +def _fresh_state(monkeypatch, tmp_path): + monkeypatch.setattr(session, "STAGE_IDLE_RETRIES", session.STAGE_IDLE_RETRIES) + monkeypatch.delenv(spend.JOURNAL_ALIAS, raising=False) + # The machine's own secrets files must not decide what a test sees masked. + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (tmp_path / "no-secrets.json",)) + monkeypatch.setattr(reading_entry, "_secret_values", None) + spend._stages.clear() + spend.journal_to(None) + yield + spend._stages.clear() + spend.journal_to(None) + + +@pytest.fixture(params=[READ_AGENT_VALUES, FIND_AGENTS_VALUES], ids=["read_agent", "find_agents"]) +def values(request): + return request.param + + +def _job(source: dict | None = None) -> dict: + return { + "job_id": "job", + "run_id": "run", + "execution": "hosted", + "source": source + or { + "kind": "github", + "visibility": "public", + "repository": "acme/agent", + "commit_sha": "a" * 40, + }, + "runtime": {"isolation": "dedicated_vm"}, + "agent": {"connector": "auto"}, + } + + +class _Work: + """A mode's work that remembers being called; ``action`` stands in for what it does.""" + + def __init__(self, code: int = 0, action=None) -> None: + self.code = code + self.action = action + self.calls: list[tuple] = [] + + async def __call__(self, job, source, output) -> int: + self.calls.append((job, source, output)) + if self.action is not None: + self.action(job, source, output) + return self.code + + +def _accept(_parsed) -> None: + return None + + +def _mode(values: dict, work, check_job=_accept, **overrides) -> ReadingMode: + return ReadingMode(check_job=check_job, work=work, **{**values, **overrides}) + + +def _run(mode: ReadingMode, root: Path, job_path: Path | None = None) -> int: + if job_path is None: + job_path = root / "job.json" + job_path.write_text(json.dumps(_job()), encoding="utf-8") + return run_reading_mode( + mode, job_path=job_path, source=root / "source", output=root / "reading" + ) + + +def _failure_in(output: Path) -> dict: + return json.loads((output / "failure.json").read_text(encoding="utf-8")) + + +def _cost_in(output: Path) -> dict: + return json.loads((output / "cost.json").read_text(encoding="utf-8")) + + +def _shape(failure: dict) -> dict: + return {key: failure[key] for key in ("domain", "stage", "code", "retryable")} + + +def _refusal(code: str = "scan_job_invalid") -> HarnessFailure: + return HarnessFailure( + domain=FailureDomain.PLATFORM_SYNC, + stage=HarnessStage.UNDERSTANDING_AGENT, + code=code, + message="A scan reads a whole repository, not one folder of it.", + retryable=False, + details={"subfolder": "agents/support"}, + ) + + +# The helper ends the process with os._exit at its limit, so those tests run it in a child. +_CHILD = r""" +import asyncio, json, sys, time +config = json.loads(sys.argv[1]) +sys.path[:0] = config["sys_path"] +from pathlib import Path +from fi.alk.harness import reading_entry, spend +from fi.alk.harness.reading_entry import ReadingMode, run_reading_mode + +reading_entry._SECRET_FILES = () +root = Path(config["root"]) + + +def broken(*_args, **_kwargs): + raise RuntimeError("this writer is broken") + + +async def work(job, source, output): + if config["work"] == "returns": + return 0 + if config["work"] == "records_then_blocks": + spend.record("read-agent", 0.21, turns=1, models={"m"}) + if config["work"] == "breaks_the_writers_then_blocks": + reading_entry.write_reading_failure = broken + reading_entry.write_reading_cost = broken + await asyncio.to_thread(time.sleep, config["block_seconds"]) + return 0 + + +mode = ReadingMode(check_job=lambda job: None, work=work, **config["mode"]) +(root / "started").write_text(repr(time.time())) +code = run_reading_mode( + mode, job_path=root / "job.json", source=root / "source", output=root / "reading" +) +time.sleep(config["linger_seconds"]) +sys.exit(code) +""" + +_BLOCK_SECONDS = 20.0 + + +def _run_in_a_child(root: Path, mode_values: dict, work: str, *, linger: float = 0.0): + """Returns the finished child and the seconds from the helper's start to the child's end.""" + root.mkdir(parents=True, exist_ok=True) + (root / "job.json").write_text(json.dumps(_job()), encoding="utf-8") + config = { + "sys_path": [entry for entry in sys.path if entry], + "root": str(root), + "mode": mode_values, + "work": work, + "block_seconds": _BLOCK_SECONDS, + "linger_seconds": linger, + } + env = { + **os.environ, + "OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:9", + "OTEL_TRACES_EXPORTER": "none", + } + env.pop(spend.JOURNAL_ALIAS, None) + finished = subprocess.run( + [sys.executable, "-c", _CHILD, json.dumps(config)], + env=env, + capture_output=True, + text=True, + timeout=_BLOCK_SECONDS + 40, + ) + ended = time.time() + started = float((root / "started").read_text(encoding="utf-8")) + return finished, ended - started + + +@pytest.mark.parametrize("case", ["missing", "not_json", "not_a_job", "folder_on_an_upload"]) +def test_an_unreadable_job_is_refused_before_any_work(tmp_path, values, case): + job_path = tmp_path / "job.json" + if case == "not_json": + job_path.write_text("{not json", encoding="utf-8") + elif case == "not_a_job": + job_path.write_text(json.dumps({"hello": "world"}), encoding="utf-8") + elif case == "folder_on_an_upload": + upload = {"kind": "archive", "archive_artifact_id": "upload-1"} + # Without the folder the same upload is a good job, so only the folder is refused. + HarnessJob.model_validate(_job(upload)) + job_path.write_text( + json.dumps(_job({**upload, "subfolder": "agents/support"})), encoding="utf-8" + ) + checked: list = [] + work = _Work() + + code = _run(_mode(values, work, check_job=checked.append), tmp_path, job_path) + + output = tmp_path / "reading" + assert code == 78 + assert _shape(_failure_in(output)) == { + "domain": "platform_sync", + "stage": "understanding_agent", + "code": READING_JOB_INVALID, + "retryable": False, + } + assert not (output / "cost.json").exists() + assert checked == [] + assert work.calls == [] + + +def test_the_modes_own_job_check_refuses_before_any_work(tmp_path, values): + seen: list = [] + + def check_job(job): + seen.append(job) + return _refusal() + + work = _Work() + + assert _run(_mode(values, work, check_job=check_job), tmp_path) == 78 + + output = tmp_path / "reading" + assert _failure_in(output) == _refusal().model_dump(mode="json") + assert [type(job) for job in seen] == [HarnessJob] + assert seen[0].job_id == "job" + assert not (output / "cost.json").exists() + assert work.calls == [] + + +def test_a_job_check_that_raises_is_a_crash_before_any_work(tmp_path, values): + def check_job(_job): + raise KeyError("metadata") + + work = _Work() + + assert _run(_mode(values, work, check_job=check_job), tmp_path) == 1 + + output = tmp_path / "reading" + failure = _failure_in(output) + assert _shape(failure) == { + "domain": "infrastructure", + "stage": "understanding_agent", + "code": values["crashed_code"], + "retryable": True, + } + assert "KeyError" in failure["message"] + assert not (output / "cost.json").exists() + assert work.calls == [] + + +def test_a_non_empty_output_is_refused_and_left_alone(tmp_path, values): + old = tmp_path / "old" + (old / "reading").mkdir(parents=True) + (old / "reading" / "contract.json").write_bytes(b'{"old": true}') + stale = tmp_path / "stale" + (stale / "reading").mkdir(parents=True) + stale_failure = b'{"code": "from_an_earlier_run"}' + (stale / "reading" / "failure.json").write_bytes(stale_failure) + a_file = tmp_path / "a_file" + a_file.mkdir() + (a_file / "reading").write_bytes(b"not a folder") + work = _Work() + + assert _run(_mode(values, work), old) == 78 + assert _run(_mode(values, work), stale) == 78 + assert _run(_mode(values, work), a_file) == 78 + + assert (old / "reading" / "contract.json").read_bytes() == b'{"old": true}' + assert sorted(entry.name for entry in (old / "reading").iterdir()) == [ + "contract.json", + "failure.json", + ] + assert _shape(_failure_in(old / "reading")) == { + "domain": "platform_sync", + "stage": "understanding_agent", + "code": READING_OUTPUT_NOT_EMPTY, + "retryable": False, + } + assert [entry.name for entry in (stale / "reading").iterdir()] == ["failure.json"] + assert (stale / "reading" / "failure.json").read_bytes() == stale_failure + assert (a_file / "reading").read_bytes() == b"not a folder" + assert work.calls == [] + + +def test_cost_starts_at_zero_and_incomplete_before_the_work(tmp_path, values): + seen: dict = {} + + def look(_job, _source, output): + seen.update(_cost_in(output)) + + assert _run(_mode(values, _Work(action=look)), tmp_path) == 0 + + assert seen["total_usd"] == 0 + assert seen["unpriced_turns"] == 0 + assert seen["stages"] == [] + assert seen["complete"] is False + + +def test_a_failed_first_cost_write_stops_before_the_work(tmp_path, values, monkeypatch): + blocked = tmp_path / "blocked" + blocked.write_text("a file, not a folder", encoding="utf-8") + assert write_reading_cost(blocked, complete=False) is False + + monkeypatch.setattr(reading_entry, "write_reading_cost", lambda output, *, complete: False) + work = _Work() + + assert _run(_mode(values, work), tmp_path) == 1 + + assert work.calls == [] + assert _shape(_failure_in(tmp_path / "reading")) == { + "domain": "infrastructure", + "stage": "understanding_agent", + "code": values["crashed_code"], + "retryable": True, + } + + +@pytest.mark.parametrize("retryable, exit_code", [(False, 78), (True, 1)]) +def test_the_time_limit_writes_its_code_and_ends_the_process( + tmp_path, values, retryable, exit_code +): + limit = 1.0 + mode_values = {**values, "time_limit_seconds": limit, "timed_out_retryable": retryable} + + finished, elapsed = _run_in_a_child(tmp_path, mode_values, "blocks") + + output = tmp_path / "reading" + assert finished.returncode == exit_code, finished.stderr + assert limit <= elapsed <= limit + 5 + assert _shape(_failure_in(output)) == { + "domain": "infrastructure", + "stage": "understanding_agent", + "code": values["timed_out_code"], + "retryable": retryable, + } + assert _cost_in(output)["complete"] is False + assert "reading started at " in finished.stderr + + +def test_the_time_limit_ends_the_process_even_when_its_writes_fail(tmp_path): + limit = 1.0 + mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": limit} + + finished, elapsed = _run_in_a_child(tmp_path, mode_values, "breaks_the_writers_then_blocks") + + assert finished.returncode == 78, finished.stderr + assert limit <= elapsed <= limit + 5 + # Proof the writers really raised: the limit could leave no failure of its own. + assert not (tmp_path / "reading" / "failure.json").exists() + + +def test_the_time_limit_does_not_fire_after_the_work_returns(tmp_path): + mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} + + finished, elapsed = _run_in_a_child(tmp_path, mode_values, "returns", linger=2.0) + + assert finished.returncode == 0, finished.stderr + assert elapsed >= 2.0 + assert not (tmp_path / "reading" / "failure.json").exists() + + +def test_a_crash_writes_its_code_with_no_secret_and_a_capped_message(tmp_path, values): + def crash(*_args): + raise RuntimeError("could not fetch https://user:pw@host/repo.git " + "x" * 3000) + + assert _run(_mode(values, _Work(action=crash)), tmp_path) == 1 + + output = tmp_path / "reading" + failure = _failure_in(output) + assert _shape(failure) == { + "domain": "infrastructure", + "stage": "understanding_agent", + "code": values["crashed_code"], + "retryable": True, + } + assert failure["message"].startswith( + "RuntimeError: could not fetch https://user:***@host/repo.git " + ) + assert "pw" not in failure["message"] + assert MESSAGE_LIMIT == 2000 + assert len(failure["message"]) == MESSAGE_LIMIT + assert _cost_in(output)["complete"] is False + + blocked = tmp_path / "blocked" + blocked.write_text("a file, not a folder", encoding="utf-8") + assert write_reading_failure(_refusal(), blocked) is None + + +def test_a_value_from_the_secrets_file_is_masked(tmp_path, monkeypatch, values): + target = "sk-0123456789abcdefg" + simulator = "sim-0123456789abcdef" + assert len(target) == len(simulator) == 20 + secrets = tmp_path / "secrets.json" + secrets.write_text(json.dumps({"TARGET_KEY": target}), encoding="utf-8") + simulator_secrets = tmp_path / "simulator-secrets.json" + simulator_secrets.write_text(json.dumps({"SIMULATOR_KEY": simulator}), encoding="utf-8") + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (secrets, simulator_secrets)) + + def crash(*_args): + raise RuntimeError(f"the provider refused {target} and {simulator}") + + assert _run(_mode(values, _Work(action=crash)), tmp_path) == 1 + + assert _failure_in(tmp_path / "reading")["message"] == ( + "RuntimeError: the provider refused *** and ***" + ) + assert _DEFAULT_SECRET_FILES == ( + Path("/run/futureagi/secrets.json"), + Path("/run/futureagi/simulator-secrets.json"), + ) + + +@pytest.mark.parametrize("exit_code", [0, 1, 78]) +def test_the_works_exit_code_is_returned(tmp_path, values, exit_code): + assert _run(_mode(values, _Work(code=exit_code)), tmp_path) == exit_code + assert not (tmp_path / "reading" / "failure.json").exists() + + +def test_a_works_own_incomplete_cost_is_kept(tmp_path, values): + def account(_job, _source, output): + spend.record("read-agent", 0.42, turns=1, models={"m"}) + assert write_reading_cost(output, complete=False) + # Spend the work's file leaves out: any rewrite after the work returns would add it. + spend.record("read-agent", 0.08, turns=1, models={"m"}) + + assert _run(_mode(values, _Work(action=account)), tmp_path) == 0 + + cost = _cost_in(tmp_path / "reading") + assert cost["total_usd"] == 0.42 + assert cost["complete"] is False + + +def test_cost_is_left_after_a_timeout_and_a_crash(tmp_path): + def record_then_crash(*_args): + spend.record("read-agent", 0.21, turns=1, models={"m"}) + raise RuntimeError("the reading broke") + + crashed = tmp_path / "crash" + crashed.mkdir() + assert _run(_mode(READ_AGENT_VALUES, _Work(action=record_then_crash)), crashed) == 1 + cost = _cost_in(crashed / "reading") + assert cost["total_usd"] == 0.21 + assert cost["complete"] is False + assert _failure_in(crashed / "reading")["code"] == "agent_reading_crashed" + + timed_out = tmp_path / "timeout" + mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} + finished, _ = _run_in_a_child(timed_out, mode_values, "records_then_blocks") + assert finished.returncode == 78, finished.stderr + cost = _cost_in(timed_out / "reading") + assert cost["total_usd"] == 0.21 + assert cost["complete"] is False + assert _failure_in(timed_out / "reading")["code"] == "agent_reading_timed_out" + + +def test_the_checks_run_in_order(tmp_path, values): + unreadable = tmp_path / "unreadable" + (unreadable / "reading").mkdir(parents=True) + (unreadable / "reading" / "old.txt").write_text("old", encoding="utf-8") + bad_job = unreadable / "job.json" + bad_job.write_text("{not json", encoding="utf-8") + refused = tmp_path / "refused" + (refused / "reading").mkdir(parents=True) + (refused / "reading" / "old.txt").write_text("old", encoding="utf-8") + work = _Work() + + assert _run(_mode(values, work), unreadable, bad_job) == 78 + assert _run(_mode(values, work, check_job=lambda _parsed: _refusal()), refused) == 78 + + assert _failure_in(unreadable / "reading")["code"] == READING_JOB_INVALID + assert _failure_in(refused / "reading")["code"] == "scan_job_invalid" + assert not (unreadable / "reading" / "cost.json").exists() + assert not (refused / "reading" / "cost.json").exists() + assert work.calls == [] + + +def test_cost_json_keeps_the_build_keys_and_adds_complete(tmp_path): + spend.record( + "read-agent", + 0.125, + turns=3, + models={"claude-x"}, + tokens_in=1000, + tokens_out=200, + tokens_cached=100, + tools={"submit_contract": 1}, + refusals={"agent_type": 1}, + ) + output = tmp_path / "reading" + + assert write_reading_cost(output, complete=True) is True + + snapshot = spend.snapshot() + assert "complete" not in snapshot + assert _cost_in(output) == {**snapshot, "complete": True} + assert [entry.name for entry in output.iterdir()] == ["cost.json"] + + +def test_an_unpriced_turn_marks_the_cost_incomplete(tmp_path): + spend.record("read-agent", 0.1, turns=1, models={"m"}) + spend.record("read-agent", None, turns=1, models={"m"}) + output = tmp_path / "reading" + + assert write_reading_cost(output, complete=True) is True + + cost = _cost_in(output) + assert cost["unpriced_turns"] == 1 + assert cost["total_usd"] == 0.1 + assert cost["complete"] is False From 0d70469528c19a3e50b1e1ebad2f759b2d0ce69f Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Fri, 6 Nov 2026 14:39:26 +0530 Subject: [PATCH 02/23] feat(harness): list the agents in a repo without building it Adds --find-agents to the authoring entrypoint, run through the shared reading helper. It writes /work/reading/agents.json in the shape the platform's AgentScanSerializer reads, plus cost.json on every return. - A plain code walk finds candidates (framework builds, langgraph.json graphs, agent.py, provider clients) and joins files into agents by their imports, without running anything. - Each agent's folder gets TH-8805's framework answer, with a build's limits, in one child process the scan's clock can stop. - A model is asked only when the walk finds nothing usable (mostly TypeScript and JavaScript repos). Its read/glob/grep tools stay inside the source, and only the paths it names, once checked, are kept. - Names, evidence and paths are built to fit the platform's rules, and the file is checked against a copy of them before it is written. - An agent at a path the platform's rule can't carry (a Next.js app/(chat)/ route, say) is left out with truncated set, and is reported as agent_path_unsupported when nothing else is found. TH-8950 Signed-off-by: khushalsonawat --- .github/workflows/sdk-smoke.yml | 6 + src/fi/alk/harness/agent_scan.py | 547 +++++++ src/fi/alk/harness/authoring_entrypoint.py | 18 +- src/fi/alk/harness/scan_ai.py | 969 ++++++++++++ src/fi/alk/harness/scan_document.py | 666 ++++++++ src/fi/alk/harness/scan_folders.py | 213 +++ src/fi/alk/harness/scan_walk.py | 1382 +++++++++++++++++ .../alk/harness/skills/find-agents/SKILL.md | 86 + .../fixtures/agent_scan_e07d56fcd.json | 625 ++++++++ tests/harness/test_agent_scan.py | 1175 ++++++++++++++ .../test_authoring_entrypoint_find_agents.py | 242 +++ tests/harness/test_scan_ai.py | 1007 ++++++++++++ tests/harness/test_scan_document.py | 605 ++++++++ tests/harness/test_scan_folders.py | 222 +++ tests/harness/test_scan_walk.py | 1201 ++++++++++++++ 15 files changed, 8963 insertions(+), 1 deletion(-) create mode 100644 src/fi/alk/harness/agent_scan.py create mode 100644 src/fi/alk/harness/scan_ai.py create mode 100644 src/fi/alk/harness/scan_document.py create mode 100644 src/fi/alk/harness/scan_folders.py create mode 100644 src/fi/alk/harness/scan_walk.py create mode 100644 src/fi/alk/harness/skills/find-agents/SKILL.md create mode 100644 tests/harness/fixtures/agent_scan_e07d56fcd.json create mode 100644 tests/harness/test_agent_scan.py create mode 100644 tests/harness/test_authoring_entrypoint_find_agents.py create mode 100644 tests/harness/test_scan_ai.py create mode 100644 tests/harness/test_scan_document.py create mode 100644 tests/harness/test_scan_folders.py create mode 100644 tests/harness/test_scan_walk.py diff --git a/.github/workflows/sdk-smoke.yml b/.github/workflows/sdk-smoke.yml index 9777af0dd..2c8a33a27 100644 --- a/.github/workflows/sdk-smoke.yml +++ b/.github/workflows/sdk-smoke.yml @@ -104,6 +104,12 @@ jobs: tests/harness/test_executor_completed_counts.py tests/harness/test_reading_entry.py tests/harness/test_agent_reading.py + tests/harness/test_scan_walk.py + tests/harness/test_scan_folders.py + tests/harness/test_scan_document.py + tests/harness/test_scan_ai.py + tests/harness/test_agent_scan.py + tests/harness/test_authoring_entrypoint_find_agents.py tests/test_bump_deployment_values.py tests/test_release_workflow.py tests/test_release_pr_step.py diff --git a/src/fi/alk/harness/agent_scan.py b/src/fi/alk/harness/agent_scan.py new file mode 100644 index 000000000..5e6a0153b --- /dev/null +++ b/src/fi/alk/harness/agent_scan.py @@ -0,0 +1,547 @@ +"""Finds the agents in a repository at one commit, or in an upload, and lists them for the platform. + +The platform shows this list before it creates an agent, and bills the scan from ``cost.json``. +A plain code check runs first and costs nothing. The model is asked only when that check finds +nothing outside test and example folders, or only files that call a model directly beside +TypeScript or JavaScript it can't read, because the customer pays for every turn of it. So +once the model has answered, nothing it was paid for is thrown away: a crash after it still +writes what was kept, and never ends in a retryable failure that would pay for the same answer +again. Every return leaves the final ``cost.json``, marked incomplete whenever the model's turn +may be missing from it. +""" + +from __future__ import annotations + +import logging +import math +import os +import stat +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field, replace +from pathlib import Path + +from .job import FailureDomain, HarnessFailure, HarnessJob, HarnessStage, SourceKind +from .reading_entry import ReadingMode, write_reading_cost, write_reading_failure +from .scan_ai import AiOutcome, run_ai_step +from .scan_document import Row, ScanResultInvalid, finish, root_name, write_agents_json +from .scan_folders import ClockRanOut, FolderChecker, FolderCheckerBroken +from .scan_walk import _WEAKEST, Candidate, Hazard, WalkResult, describe_entry, walk_repo + +logger = logging.getLogger(__name__) + +# Failures, written to failure.json. +SCAN_JOB_INVALID = "scan_job_invalid" +SCAN_SOURCE_MISSING = "scan_source_missing" +SCAN_CRASHED = "scan_crashed" +SCAN_HARD_LIMIT_REACHED = "scan_hard_limit_reached" +SCAN_MODEL_ERROR = "scan_model_error" +SCAN_MODEL_FORBIDDEN = "scan_model_forbidden" +SCAN_RESULT_INVALID = "scan_result_invalid" + +# Results, written to agents.json as its error. +SCAN_LIMIT_REACHED = "scan_limit_reached" +SCAN_TIMED_OUT = "scan_timed_out" +SCAN_OUT_OF_MEMORY = "scan_out_of_memory" +SCAN_AI_NO_ANSWER = "scan_ai_no_answer" +SCAN_FAILED_AFTER_AI = "scan_failed_after_ai" +AGENT_PATH_UNSUPPORTED = "agent_path_unsupported" + +_EXIT_DONE = 0 +_EXIT_RETRYABLE = 1 +_EXIT_REFUSED = 78 + + +def _seconds(name: str, default: float) -> float: + try: + value = float(os.environ.get(name, "")) + except ValueError: + return default + return value if math.isfinite(value) and value > 0 else default + + +CLOCK = _seconds("ALK_SCAN_MAX_SECONDS", 1200.0) +AI_SECONDS = _seconds("ALK_SCAN_AI_MAX_SECONDS", 600.0) +# The helper's hard stop comes this long after the scan's own clock, so a scan that stops itself +# always writes its own file first. +HARD_LIMIT_MARGIN = 600.0 +# The model is asked only with this much left: its own time, then what follows it. +AI_LEAST_LEFT = 360.0 +# Kept back from the model for the stop sequence, the AI rows' descriptions and their checks. +AI_RESERVE = 180.0 + +_ERROR_MESSAGES = { + SCAN_LIMIT_REACHED: "The repository is larger than the scan reads, and no agent was found " + "before the limit.", + SCAN_TIMED_OUT: "The scan ran out of time before it found an agent.", + SCAN_OUT_OF_MEMORY: "The scan ran out of memory before it found an agent.", + SCAN_AI_NO_ANSWER: "The model looked for agents and gave no answer.", + SCAN_FAILED_AFTER_AI: "The scan failed after the model looked for agents, and nothing it " + "found could be kept.", + AGENT_PATH_UNSUPPORTED: "Every agent found sits at a path the platform cannot use.", +} +_STOPPED = {"limit": SCAN_LIMIT_REACHED, "clock": SCAN_TIMED_OUT, "memory": SCAN_OUT_OF_MEMORY} + +# A walk that ran out of memory is dropped whole, so nothing it decided is half kept. +_NO_WALK = WalkResult( + agents=(), + more_than_fifty=False, + stopped="memory", + unread_files=0, + source_outside_side=False, + candidate_outside_side=False, + script_outside_side=False, + only_weak_outside_side=False, + paths_left_out=0, + upload_others=0, + upload_others_side=0, +) + + +@dataclass +class _Scan: + """What one scan has so far, kept where the after-model catch can still reach it.""" + + upload: bool + walk_rows: list[Row] = field(default_factory=list) + ai_found: list[Candidate] = field(default_factory=list) + ai_rows: list[Row] = field(default_factory=list) + truncated: bool = False + cut_by_clock: bool = False + complete: bool = True + # One look per folder: many agents can share one, and the look walks skipped folders. + hazards: dict[str, Hazard] = field(default_factory=dict) + # The model was asked past walk rows that only call a model, so the cap must not drop + # what it found in their favour. + models_first: bool = False + + +def _failure(domain: FailureDomain, code: str, message: str, *, retryable: bool) -> HarnessFailure: + return HarnessFailure( + domain=domain, + stage=HarnessStage.UNDERSTANDING_AGENT, + code=code, + message=message, + retryable=retryable, + ) + + +def _fail( + output: Path, domain: FailureDomain, code: str, message: str, *, retryable: bool +) -> int: + write_reading_failure(_failure(domain, code, message, retryable=retryable), output) + logger.warning("agent scan: failed, code=%s retryable=%s", code, retryable) + return _EXIT_RETRYABLE if retryable else _EXIT_REFUSED + + +def _check_job(job: HarnessJob) -> HarnessFailure | None: + source = job.source + if source.kind in (SourceKind.REMOTE, SourceKind.PROVIDER) or source.subfolder: + return _failure( + FailureDomain.PLATFORM_SYNC, + SCAN_JOB_INVALID, + "The agent scan reads a whole GitHub repository or an upload; this job names a " + "remote or provider agent, or a folder.", + retryable=False, + ) + return None + + +def _holds_a_file(source: Path) -> bool: + """Whether ``source`` is a folder with at least one regular file, stopping at the first.""" + try: + if not source.is_dir(): + return False + except OSError: + return False + for directory, _, files in os.walk(source, followlinks=False): + for name in files: + try: + if stat.S_ISREG(os.lstat(os.path.join(directory, name)).st_mode): + return True + except OSError: + continue + return False + + +def _hints(walk: WalkResult) -> str: + """What the walk counted, for the model's opening. Counts only, never a path: a path invited + the model to submit again what the walk already listed.""" + side = sum(candidate.side for candidate in walk.agents) + walk.upload_others_side + # The model is asked past candidates outside side folders only when every one of them + # only calls a model, so each unlisted one outside them is one of those. + weak = sum( + not candidate.side and candidate.sign in _WEAKEST for candidate in walk.agents + ) + (walk.upload_others - walk.upload_others_side) + return ( + "source files outside those folders: yes; " + f"agent-like files inside them: {side}; " + f"files outside them that only call a model: {weak}; " + f"files the check could not read: {walk.unread_files}." + ) + + +def _unchecked(candidate: Candidate, *, found_by_ai: bool = True) -> Row: + return Row( + candidate=candidate, + framework={"name": "unknown", "version": None, "label": None}, + finished=False, + found_by_ai=found_by_ai, + ) + + +def _check_into( + rows: list[Row], + checker: FolderChecker, + source: Path, + candidates: Sequence[Candidate], + scan: _Scan, + *, + found_by_ai: bool, + deadline: float, + clock: Callable[[], float], +) -> None: + """Checks each candidate's folder in order, appending each row as soon as it is known.""" + for index, candidate in enumerate(candidates): + try: + word = checker.check(source, candidate.subfolder, deadline=deadline, clock=clock) + except ClockRanOut: + scan.truncated = scan.cut_by_clock = True + # Found, so listed without its folder's word rather than lost: the person can still + # pick it, and a build of it names the framework. + rows.extend( + _unchecked(left, found_by_ai=found_by_ai) for left in candidates[index:] + ) + return + rows.append(Row(candidate, word.framework, word.finished, found_by_ai)) + + +def _one_row(walk_rows: Sequence[Row], ai_rows: Sequence[Row]) -> tuple[list[Row], bool]: + """An upload's one row, at the root, and whether any other was left out.""" + every = [*walk_rows, *ai_rows] + if not every: + return [], False + outside = next((row for row in walk_rows if not row.candidate.side), None) + if outside is not None and outside.candidate.sign in _WEAKEST: + # The model was asked past it, so what the model found outside side folders is + # likelier the agent than a file that only calls a model. + outside = next((row for row in ai_rows if not row.candidate.side), outside) + chosen = ( + outside + or next((row for row in ai_rows if not row.candidate.side), None) + or every[0] + ) + if chosen.candidate.subfolder: + chosen = replace(chosen, candidate=replace(chosen.candidate, subfolder="")) + return [chosen], len(every) > 1 + + +def _write( + job: HarnessJob, + output: Path, + rows: Sequence[Row], + *, + upload: bool, + truncated: bool, + error: str | None, + models_first: bool, +) -> int: + try: + body, _ = finish( + rows=rows, + root_name=root_name(job), + commit_sha=None if upload else job.source.commit_sha, + truncated=truncated, + error=None if error is None else (error, _ERROR_MESSAGES[error]), + models_first=models_first, + ) + except ScanResultInvalid as invalid: + logger.error("agent scan: result invalid, problem=%s", invalid.problem) + return _fail( + output, + FailureDomain.INFRASTRUCTURE, + SCAN_RESULT_INVALID, + "The agent list broke the platform's rules and was not written; this is a fault " + "in the scan, not in the repository.", + retryable=False, + ) + write_agents_json(output, body) + logger.info( + "agent scan: written, rows=%d truncated=%s error=%s", + len(rows), + truncated, + error or "none", + ) + return _EXIT_DONE + + +def _result( + job: HarnessJob, + output: Path, + scan: _Scan, + walk: WalkResult, + outcome: AiOutcome | None, +) -> int: + """Steps 6.4 to 9: the one row of an upload, the model's outcome, then the file.""" + if scan.upload: + rows, others = _one_row(scan.walk_rows, scan.ai_rows) + else: + rows, others = [*scan.walk_rows, *scan.ai_rows], False + truncated = scan.truncated or others + error: str | None = None + if outcome is not None: + if outcome.kind == "model_error": + logger.warning("agent scan: model step failed, %s", outcome.message) + if outcome.retryable: + return _fail( + output, + FailureDomain.INFRASTRUCTURE, + SCAN_MODEL_ERROR, + "The model failed while looking for agents; a retry may get through.", + retryable=True, + ) + if not rows: + return _fail( + output, + FailureDomain.INFRASTRUCTURE, + SCAN_MODEL_ERROR, + "The model refused the request to look for agents; a retry would be " + "refused the same way.", + retryable=False, + ) + # The walk's rows are already checked, and a retry would be refused again. + truncated = True + elif outcome.kind == "forbidden": + return _fail( + output, + FailureDomain.INFRASTRUCTURE, + SCAN_MODEL_FORBIDDEN, + "The provider billed a model this kit may not spend on, so the scan stopped.", + retryable=False, + ) + elif outcome.kind in ("no_answer", "timed_out"): + if rows: + truncated = True + else: + error = SCAN_AI_NO_ANSWER if outcome.kind == "no_answer" else SCAN_TIMED_OUT + truncated = truncated or outcome.refused > 0 or outcome.uncarried > 0 + truncated = ( + truncated + or walk.more_than_fifty + or bool(walk.stopped and rows) + or walk.paths_left_out > 0 + or walk.unread_files > 0 + or scan.cut_by_clock + or walk.upload_others > 0 + or any(row.candidate.partial or not row.finished for row in rows) + ) + if not rows and error is None: + if walk.stopped: + error = _STOPPED[walk.stopped] + elif walk.paths_left_out or (outcome is not None and outcome.uncarried): + error = AGENT_PATH_UNSUPPORTED + return _write( + job, + output, + rows, + upload=scan.upload, + truncated=truncated, + error=error, + models_first=scan.models_first, + ) + + +def _after_ai( + job: HarnessJob, + source: Path, + output: Path, + scan: _Scan, + walk: WalkResult, + outcome: AiOutcome, + checker: FolderChecker, + *, + deadline: float, + clock: Callable[[], float], +) -> int: + for folder, entry in outcome.pairs: + candidate = describe_entry( + source, + entry, + folder, + deadline=deadline, + upload=scan.upload, + clock=clock, + hazards=scan.hazards, + ) + if candidate is None: + scan.truncated = True + continue + scan.ai_found.append(candidate) + # Walk rows first, so a walk row always beats a model row for the same entry, and a + # duplicate is never checked twice. + taken = {row.candidate.entry for row in scan.walk_rows} + fresh: list[Candidate] = [] + for candidate in scan.ai_found: + if candidate.entry not in taken: + taken.add(candidate.entry) + fresh.append(candidate) + scan.ai_found = fresh + _check_into( + scan.ai_rows, + checker, + source, + fresh, + scan, + found_by_ai=True, + deadline=deadline, + clock=clock, + ) + return _result(job, output, scan, walk, outcome) + + +def _keep_what_was_paid_for( + job: HarnessJob, output: Path, scan: _Scan, crash: Exception +) -> int: + """After the model was paid for, a crash still ends in a file, never in a paid retry.""" + # The class only: a message could carry a path or a value from the repository. + logger.error("agent scan: failed after the model step, class=%s", type(crash).__name__) + checked = {row.candidate.entry for row in scan.ai_rows} + ai_rows = [ + *scan.ai_rows, + *(_unchecked(candidate) for candidate in scan.ai_found if candidate.entry not in checked), + ] + if scan.upload: + rows, _ = _one_row(scan.walk_rows, ai_rows) + else: + rows = [*scan.walk_rows, *ai_rows] + try: + body, _ = finish( + rows=rows, + root_name=root_name(job), + commit_sha=None if scan.upload else job.source.commit_sha, + truncated=True, + error=None + if rows + else (SCAN_FAILED_AFTER_AI, _ERROR_MESSAGES[SCAN_FAILED_AFTER_AI]), + models_first=scan.models_first, + ) + write_agents_json(output, body) + return _EXIT_DONE + except Exception as failed: + logger.error("agent scan: the kept rows could not be written, class=%s", type(failed).__name__) + try: + body, _ = finish( + rows=[], + root_name=root_name(job), + commit_sha=None if scan.upload else job.source.commit_sha, + truncated=True, + error=(SCAN_FAILED_AFTER_AI, _ERROR_MESSAGES[SCAN_FAILED_AFTER_AI]), + ) + write_agents_json(output, body) + return _EXIT_DONE + except Exception as failed: + logger.error("agent scan: the fixed error could not be written, class=%s", type(failed).__name__) + raise crash + + +async def _scan(job: HarnessJob, source: Path, output: Path, scan: _Scan) -> int: + clock = time.monotonic + # Taken first, so the source check counts against the clock too. + deadline = clock() + CLOCK + if not _holds_a_file(source): + return _fail( + output, + FailureDomain.PLATFORM_SYNC, + SCAN_SOURCE_MISSING, + "The source to scan is missing, is not a folder, or holds no file.", + retryable=True, + ) + try: + walk = walk_repo( + source, + deadline=deadline, + upload=scan.upload, + clock=clock, + hazards=scan.hazards, + ) + except MemoryError: + walk = _NO_WALK + logger.warning("agent scan: the walk ran out of memory and was dropped") + with FolderChecker() as checker: + try: + _check_into( + scan.walk_rows, + checker, + source, + walk.agents, + scan, + found_by_ai=False, + deadline=deadline, + clock=clock, + ) + except FolderCheckerBroken: + # No model has run yet, so a retry costs nothing that was paid for. + return _fail( + output, + FailureDomain.INFRASTRUCTURE, + SCAN_CRASHED, + "The folder check could not run, before any model was asked.", + retryable=True, + ) + # A Python file that only calls a model says nothing of an agent written in TypeScript + # or JavaScript beside it, which the walk can't list. + asks_the_model = ( + ( + not walk.candidate_outside_side + or (walk.only_weak_outside_side and walk.script_outside_side) + ) + and walk.source_outside_side + and walk.stopped == "" + and deadline - clock() >= AI_LEAST_LEFT + ) + if not asks_the_model: + return _result(job, output, scan, walk, None) + scan.models_first = walk.candidate_outside_side + outcome = await run_ai_step( + source, + hints=_hints(walk), + seconds=min(AI_SECONDS, deadline - clock() - AI_RESERVE), + on_turn_end=lambda: write_reading_cost(output, complete=False), + ) + if not outcome.complete: + scan.complete = False + # `_write` has already turned a ScanResultInvalid into its 78, so it never reaches here. + try: + return _after_ai( + job, + source, + output, + scan, + walk, + outcome, + checker, + deadline=deadline, + clock=clock, + ) + except Exception as crash: + return _keep_what_was_paid_for(job, output, scan, crash) + + +async def _work(job: HarnessJob, source: Path, output: Path) -> int: + scan = _Scan(upload=job.source.kind is SourceKind.ARCHIVE) + try: + return await _scan(job, Path(source), Path(output), scan) + finally: + # The one place the final cost is written, so no return can leave the helper's zero. + write_reading_cost(output, complete=scan.complete) + + +FIND_AGENTS = ReadingMode( + name="find_agents", + check_job=_check_job, + work=_work, + time_limit_seconds=CLOCK + HARD_LIMIT_MARGIN, + timed_out_code=SCAN_HARD_LIMIT_REACHED, + timed_out_retryable=False, + crashed_code=SCAN_CRASHED, +) diff --git a/src/fi/alk/harness/authoring_entrypoint.py b/src/fi/alk/harness/authoring_entrypoint.py index 07f4341e1..e6ab6f416 100644 --- a/src/fi/alk/harness/authoring_entrypoint.py +++ b/src/fi/alk/harness/authoring_entrypoint.py @@ -679,11 +679,17 @@ def main(argv: list[str] | None = None, *, validate_runtime: bool = False) -> in type=Path, help="Directory the sealed bundle is written to in --certify-only mode", ) - parser.add_argument( + reading = parser.add_mutually_exclusive_group() + reading.add_argument( "--read-agent-only", action="store_true", help="Read the agent and leave its description in --output; build nothing", ) + reading.add_argument( + "--find-agents", + action="store_true", + help="Find the agents in the source and leave their list in --output; build nothing", + ) args = parser.parse_args(argv) if args.certify_only: if args.bundle_output is None: @@ -699,6 +705,16 @@ def main(argv: list[str] | None = None, *, validate_runtime: bool = False) -> in source=args.source.resolve(), output=args.output.resolve(), ) + if args.find_agents: + from .agent_scan import FIND_AGENTS + from .reading_entry import run_reading_mode + + return run_reading_mode( + FIND_AGENTS, + job_path=args.job, + source=args.source.resolve(), + output=args.output.resolve(), + ) job = HarnessJob.model_validate(json.loads(args.job.read_text(encoding="utf-8"))) if refuse_unusable_folder( diff --git a/src/fi/alk/harness/scan_ai.py b/src/fi/alk/harness/scan_ai.py new file mode 100644 index 000000000..1bd42174f --- /dev/null +++ b/src/fi/alk/harness/scan_ai.py @@ -0,0 +1,969 @@ +"""Asks a model where a repository's agents are, when the plain code check found none it can trust. + +The customer pays for every turn here, so the step is shaped around the bill. It says one thing, +once: a second message in a Claude session re-reports the first one's cost, and a reminder after +a turn limit is a fresh paid turn. An answer the model has submitted is kept whatever happens +after it, because it was paid for. The time limit always ends in the same stop sequence, so the +cost of a call in flight is either recorded or marked incomplete, never silently dropped. + +The model reads the repository through three tools of this module's own, not the kit's +general file tools. Those search with a regular expression over whole files with no bound, +which one minified line or one named pipe turns into a stalled event loop. These search for +plain text, bound every call by entries, bytes, characters and seconds, and never show a file +the kit treats as a secret. +""" + +from __future__ import annotations + +import asyncio +import fnmatch +import logging +import os +import re +import stat +import threading +from collections.abc import Callable, Iterator +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Literal + +from . import session +from .backends import SessionSpec, StageDone, ToolServer, ToolSpec +from .backends.files import SECRET_REFUSAL, above_secrets, secret_path +from .config import chosen_model, load_skill +from .framework_detection import _ALWAYS_SKIPPED, _TOP_ONLY_SKIPS +from .scan_walk import fits_path_rule +from .session import Stage, StageIdleTimeout, Turn + +logger = logging.getLogger(__name__) + +STAGE_NAME = "find_agents" +MAX_TURNS = 40 +IDLE_SECONDS = 90.0 +OPEN_SECONDS = 30.0 +# The stop sequence. Long enough for a provider to send its last bill, short enough that a +# stuck one can't eat the time the rest of the scan needs. +INTERRUPT_SECONDS = 10.0 +SPEND_WAIT_SECONDS = 60.0 +CANCEL_SECONDS = 5.0 +EXIT_SECONDS = 10.0 +TOOL_SECONDS = 20.0 + +MAX_PAIRS = 50 +MAX_ENTRIES = 2_000 +MAX_READ_BYTES = 33_554_432 +MAX_FILE_BYTES = 1_048_576 +MAX_LINE_BYTES = 65_536 +MAX_RESULT_CHARS = 8_000 +MAX_READ_LINES = 400 +# As the kit's general Read cuts a line, so a minified file can't fill a result with one line. +MAX_LINE_CHARS = 2_000 +HIT_CHARS = 300 +MAX_PATTERN_CHARS = 200 +MAX_PATH_CHARS = 4_096 +MAX_PATH_PARTS = 256 + +STOPPED_EARLY = "Stopped early at the search limit: results are incomplete. Narrow the path." +TOOLS_AFTER_SUBMIT = "Already submitted: end your turn." +# ADK's own message when a run spends its call budget, matched whole so that the phrase quoted +# inside some other error never reads as a turn limit. +_CALL_LIMIT = re.compile(r"Max number of llm calls limit of `\d+` exceeded") +SUBMIT_AFTER_SUBMIT = "Already submitted." + +_OUTSIDE = "That path is not allowed: give a path inside the repository." +_SKIPPED = "That path is in a folder this step skips: installed, built or generated files." +_MISSING = "No such file or folder in the repository." +_NOT_A_FILE = "That is not a file this tool reads: only regular files up to 1 MiB are read." +_NOT_A_FOLDER = "That is not a folder in the repository." +_BAD_NUMBER = "offset and limit must be whole numbers." +_BAD_PATTERN = "pattern must be text of 1 to 200 characters." +_BAD_TEXT = "text must be plain text of 1 to 200 characters." +_BAD_GLOB = "glob must be text of at most 200 characters." +_BAD_IGNORE_CASE = "ignore_case must be true or false." +_BAD_SUBMIT = 'Give {"agents": [{"folder": ..., "entry": ...}]} with at most 50 agents.' +_TOO_SLOW = "That call took too long and was stopped. Narrow the path." +_FAILED = "That call failed. Try a different path or argument." + +# Names the kit's secret filter doesn't cover. A source file named like `secret_manager.py` is +# hidden too; that's the price of never showing a settings file under an unusual name. +_SECRET_NAMES = ( + "*.env", + "*.env.*", + "*secret*", + "*credential*", + "id_rsa*", + "id_ed25519*", + "id_ecdsa*", + "*.tfvars", +) + +_OPENING = ( + "Find the runnable AI agents in this repository and submit them with " + "mcp__scan__submit_agents, as your instructions say. A plain code check found none outside " + "test and example folders, apart from any files that only call a model, which it has " + "already listed. What it counted: " +) + + +@dataclass(frozen=True) +class AiOutcome: + kind: Literal["pairs", "empty", "no_answer", "timed_out", "model_error", "forbidden"] + pairs: tuple[tuple[str, str], ...] + refused: int + complete: bool + retryable: bool + message: str + # Real files the model named whose path the platform's rule can't carry, counted once each + # over every submit: they are left out, so the list is not whole. + uncarried: int = 0 + + +class _Refused(Exception): + """A tool call the tool won't answer, with the fixed words the model is told.""" + + def __init__(self, message: str) -> None: + super().__init__(message) + self.message = message + + +class _ToolFailed(Exception): + pass + + +@dataclass +class _Answer: + """What the model has submitted. Shared by the submit tool and the file tools.""" + + called: bool = False + pairs: tuple[tuple[str, str], ...] = () + refused: int = 0 + uncarried: set[str] = field(default_factory=set) + + def finished(self) -> bool: + return bool(self.pairs) + + +class _ScanStage(Stage): + """A stage that keeps its one turn's raw end and says when a turn's cost is known.""" + + def __init__( + self, spec: SessionSpec, *, name: str, on_turn_end: Callable[[], None] + ) -> None: + super().__init__(spec, name=name) + self.done: list[StageDone] = [] + self._on_turn_end = on_turn_end + + def _events(self, received: Any, turn: Turn) -> list[Any]: + try: + return super()._events(received, turn) + finally: + # In a finally because the parent raises for a forbidden model after recording + # its spend, and that turn's cost still has to be noted. + if isinstance(received, StageDone): + self.done.append(received) + try: + self._on_turn_end() + except Exception as exc: + # A failed cost note must not turn a paid answer into a retried one. + logger.warning( + "find_agents: the cost note after the turn failed (%s)", + type(exc).__name__, + ) + + +# -- the run ----------------------------------------------------------------------------------- + + +async def run_ai_step( + source: Path, *, hints: str, seconds: float, on_turn_end: Callable[[], None] +) -> AiOutcome: + """Ask the model once for the repository's agents, within ``seconds``.""" + # A replayed quiet stream is a second paid opening. + session.STAGE_IDLE_RETRIES = 0 + answer = _Answer() + spec = SessionSpec( + system_prompt=load_skill("find-agents"), + servers={ + "scan_files": rooted_file_tools(source, finished=answer.finished), + "scan": _submit_server(source, answer), + }, + builtins=(), + cwd=str(source), + max_turns=MAX_TURNS, + model=chosen_model(), + ask=None, + gated=True, + thinking=False, + idle_timeout_seconds=IDLE_SECONDS, + ) + stage = _ScanStage(spec, name=STAGE_NAME, on_turn_end=on_turn_end) + try: + outcome = await _run(stage, answer, hints, seconds) + finally: + await _leave(stage) + logger.info( + "find_agents: %s, %d kept, %d refused, complete=%s", + outcome.kind, + len(outcome.pairs), + outcome.refused, + outcome.complete, + ) + return outcome + + +async def _run(stage: _ScanStage, answer: _Answer, hints: str, seconds: float) -> AiOutcome: + try: + await asyncio.wait_for(stage.__aenter__(), OPEN_SECONDS) + except Exception as exc: + return AiOutcome( + "model_error", + (), + 0, + False, + True, + f"the model session did not open ({type(exc).__name__})", + ) + task = asyncio.create_task(stage.say(_opening(hints))) + try: + finished, _ = await asyncio.wait({task}, timeout=seconds) + stopped = not finished + if stopped: + await _stop(stage, task) + failure = None if not task.done() or task.cancelled() else task.exception() + finally: + if not task.done(): + task.cancel() + return _outcome(stage, answer, stopped=stopped, failure=failure) + + +def _opening(hints: str) -> str: + return _OPENING + (hints or "nothing.") + + +async def _stop(stage: _ScanStage, task: asyncio.Task[Any]) -> None: + try: + await asyncio.wait_for(stage.interrupt_response(), INTERRUPT_SECONDS) + except Exception: + pass + # The turn's bill arrives after the interrupt, if it arrives at all. + await asyncio.wait({task}, timeout=SPEND_WAIT_SECONDS) + if not task.done(): + task.cancel() + await asyncio.wait({task}, timeout=CANCEL_SECONDS) + + +async def _leave(stage: _ScanStage) -> None: + try: + await asyncio.wait_for(stage.__aexit__(None, None, None), EXIT_SECONDS) + except Exception: + pass + + +def _outcome( + stage: _ScanStage, answer: _Answer, *, stopped: bool, failure: BaseException | None +) -> AiOutcome: + complete = bool(stage.done) and not stopped + if answer.pairs: + return AiOutcome( + "pairs", answer.pairs, answer.refused, complete, False, "", len(answer.uncarried) + ) + if stage.forbidden_models(): + return AiOutcome( + "forbidden", + (), + 0, + complete, + False, + "the provider billed a model this kit may not spend on", + ) + if stopped: + return AiOutcome( + "timed_out", (), 0, complete, False, "the model step ran out of time" + ) + if isinstance(failure, StageIdleTimeout): + return AiOutcome( + "model_error", + (), + 0, + complete, + True, + "the model stream went quiet (StageIdleTimeout)", + ) + if failure is not None: + return AiOutcome( + "model_error", + (), + 0, + complete, + True, + f"the model step failed ({type(failure).__name__})", + ) + done = stage.done[-1] if stage.done else None + if done is not None and done.api_error_status: + status = done.api_error_status + said = ( + f" with status {status}" + if isinstance(status, int) and not isinstance(status, bool) + else "" + ) + return AiOutcome( + "model_error", + (), + 0, + complete, + not _client_error(status), + f"the model call failed{said}", + ) + if done is not None and done.is_error and not _turn_limit(done): + return AiOutcome( + "model_error", (), 0, complete, True, "the model call ended in an error" + ) + if answer.called: + return AiOutcome("empty", (), 0, complete, False, "", len(answer.uncarried)) + return AiOutcome( + "no_answer", + (), + 0, + complete, + False, + "the model ended without submitting an answer", + ) + + +def _turn_limit(done: StageDone) -> bool: + if done.outcome in ("error_max_turns", "max_turns"): + return True + if not done.is_error or done.api_error_status: + return False + # Vertex-Gemini's call limit arrives as ADK's exception: failed, no status. The turn count + # can't say it: a reply with no parts isn't counted, and a status-less cut on the last allowed + # request reaches the limit without being one. ADK's fixed words are only matched, never + # repeated. + return any(_CALL_LIMIT.fullmatch(str(error)) for error in done.errors) + + +def _client_error(status: Any) -> bool: + # A request the provider refuses won't be accepted on a retry either, except for these + # three, which say "not now" rather than "not this". + return ( + isinstance(status, int) + and not isinstance(status, bool) + and 400 <= status <= 499 + and status not in (408, 409, 429) + ) + + +# -- the tools --------------------------------------------------------------------------------- + + +def _ok(text: str) -> dict[str, Any]: + return {"content": [{"type": "text", "text": text}]} + + +def _error(text: str) -> dict[str, Any]: + return {"content": [{"type": "text", "text": text}], "is_error": True} + + +async def _in_thread(work: Callable[..., Any], *args: Any) -> Any: + """``work(*args)`` in a daemon thread, so a stalled disk can hold neither the loop nor exit.""" + loop = asyncio.get_running_loop() + answer = loop.create_future() + + def settle(result: Any, error: BaseException | None) -> None: + # The wait below may have given up and cancelled the future already. + if answer.done(): + return + if error is None: + answer.set_result(result) + else: + answer.set_exception(error) + + def run() -> None: + result: Any = None + error: BaseException | None = None + try: + result = work(*args) + except _Refused as refused: + error = refused + except BaseException: + error = _ToolFailed() + try: + loop.call_soon_threadsafe(settle, result, error) + except RuntimeError: + pass + + threading.Thread(target=run, name="find-agents-tool", daemon=True).start() + return await asyncio.wait_for(answer, TOOL_SECONDS) + + +async def _answered(work: Callable[..., str], *args: Any) -> dict[str, Any]: + try: + return _ok(await _in_thread(work, *args)) + except _Refused as refused: + return _error(refused.message) + except asyncio.TimeoutError: + return _error(_TOO_SLOW) + except Exception: + return _error(_FAILED) + + +def rooted_file_tools( + root: Path, *, finished: Callable[[], bool] | None = None +) -> ToolServer: + """``read``, ``glob`` and ``grep``, confined to ``root`` and bounded on every call. + + ``finished`` says the answer is in; from then on the tools read nothing. ``None`` means + the answer is never in. + """ + base = os.path.realpath(str(root)) + + async def guarded(work: Callable[[str, dict[str, Any]], str], arguments: Any) -> dict[str, Any]: + try: + if finished is not None and finished(): + return _error(TOOLS_AFTER_SUBMIT) + return await _answered(work, base, arguments) + except Exception: + return _error(_FAILED) + + # Each looks its worker up when called, not when built. + async def read(arguments: dict[str, Any]) -> dict[str, Any]: + return await guarded(_read_lines, arguments) + + async def glob(arguments: dict[str, Any]) -> dict[str, Any]: + return await guarded(_glob_files, arguments) + + async def grep(arguments: dict[str, Any]) -> dict[str, Any]: + return await guarded(_grep_files, arguments) + + return ToolServer( + name="scan_files", + tools=[ + ToolSpec( + name="read", + description=( + "Read one file of the repository. Arguments: path (relative to the " + "repository's root), optional offset (first line, from 1) and limit (at " + "most 400 lines)." + ), + input_schema={ + "type": "object", + "properties": { + "path": {"type": "string"}, + "offset": {"type": "integer"}, + "limit": {"type": "integer"}, + }, + "required": ["path"], + }, + handler=read, + ), + ToolSpec( + name="glob", + description=( + "List the repository's files whose path or name matches a pattern such as " + "*.py or **/agent*.ts. Arguments: pattern, optional path of a folder to " + "look under." + ), + input_schema={ + "type": "object", + "properties": { + "pattern": {"type": "string"}, + "path": {"type": "string"}, + }, + "required": ["pattern"], + }, + handler=glob, + ), + ToolSpec( + name="grep", + description=( + "Find lines containing text, matched exactly as written: plain text, not a " + "regular expression. Arguments: text, optional path (a file or folder), glob " + "(file names to search, such as *.ts) and ignore_case." + ), + input_schema={ + "type": "object", + "properties": { + "text": {"type": "string"}, + "path": {"type": "string"}, + "glob": {"type": "string"}, + "ignore_case": {"type": "boolean"}, + }, + "required": ["text"], + }, + handler=grep, + ), + ], + ) + + +def _secret_name(name: str) -> bool: + lowered = name.lower() + return any(fnmatch.fnmatchcase(lowered, pattern) for pattern in _SECRET_NAMES) + + +def _skipped_folder(name: str, *, at_top: bool) -> bool: + # The same folders the framework check skips, so the model and the code look at one tree. + return name in _ALWAYS_SKIPPED or (at_top and name in _TOP_ONLY_SKIPS) + + +def _in_skipped_folder(folders: list[str]) -> bool: + return any( + _skipped_folder(name, at_top=index == 0) for index, name in enumerate(folders) + ) + + +def _usable(status: os.stat_result) -> bool: + # A hard link can make a file outside the source look like one inside it. + return ( + stat.S_ISREG(status.st_mode) + and status.st_nlink == 1 + and status.st_size <= MAX_FILE_BYTES + ) + + +def _under(path: str, root: str) -> bool: + return path == root or path.startswith(root.rstrip(os.sep) + os.sep) + + +def _place(root: str, raw: Any) -> tuple[str, str]: + """The model's ``path`` as (the path as written, joined to the root; where it resolves).""" + if raw is None or raw == "": + raw = "." + if ( + not isinstance(raw, str) + or "\0" in raw + or len(raw) > MAX_PATH_CHARS + or len(raw.split("/")) > MAX_PATH_PARTS + ): + raise _Refused(_OUTSIDE) + joined = raw if os.path.isabs(raw) else os.path.join(root, raw) + real = os.path.realpath(joined) + if not _under(real, root): + raise _Refused(_OUTSIDE) + if secret_path(raw, Path(root)) or above_secrets(raw, Path(root)): + raise _Refused(SECRET_REFUSAL) + if real != root: + parts = os.path.relpath(real, root).split(os.sep) + # Refused exactly where the walk would never have shown it. + if any(_secret_name(part) for part in parts): + raise _Refused(SECRET_REFUSAL) + if _in_skipped_folder(parts): + raise _Refused(_SKIPPED) + return joined, real + + +def _hidden(path: str, name: str) -> bool: + return _secret_name(name) or secret_path(path) + + +@dataclass +class _Budget: + entries: int = 0 + read_bytes: int = 0 + stopped: bool = False + + +def _listed(folder: str, budget: _Budget) -> list[str]: + names: list[str] = [] + with os.scandir(folder) as entries: + for entry in entries: + if budget.entries >= MAX_ENTRIES: + budget.stopped = True + break + budget.entries += 1 + names.append(entry.name) + return sorted(names) + + +def _walk(root: str, start: str, budget: _Budget) -> Iterator[tuple[str, str]]: + """Every usable file under ``start``, as (its path, its path relative to ``root``).""" + stack = [start] + while stack and not budget.stopped: + folder = stack.pop() + try: + names = _listed(folder, budget) + except OSError: + continue + below: list[str] = [] + for name in names: + path = os.path.join(folder, name) + try: + status = os.lstat(path) + except OSError: + continue + if _hidden(path, name): + continue + if stat.S_ISDIR(status.st_mode): + if not _skipped_folder(name, at_top=folder == root): + below.append(path) + elif _usable(status) and not name.lower().endswith(".min.js"): + yield path, os.path.relpath(path, root) + stack.extend(reversed(below)) + + +def _read_bytes(path: str) -> bytes: + # O_NONBLOCK so that a file swapped for a named pipe since its lstat can't hang the thread. + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + try: + if not _usable(os.fstat(descriptor)): + raise _Refused(_NOT_A_FILE) + chunks: list[bytes] = [] + remaining = MAX_FILE_BYTES + 1 + while remaining > 0: + chunk = os.read(descriptor, remaining) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + finally: + os.close(descriptor) + if len(data) > MAX_FILE_BYTES: + raise _Refused(_NOT_A_FILE) + return data + + +def _one_file(joined: str) -> None: + try: + status = os.lstat(joined) + except FileNotFoundError: + raise _Refused(_MISSING) from None + if not _usable(status) or os.path.basename(joined).lower().endswith(".min.js"): + raise _Refused(_NOT_A_FILE) + + +def _whole(value: Any, default: int) -> int: + if value is None: + return default + if isinstance(value, int) and not isinstance(value, bool): + return value + if isinstance(value, str) and value.isdecimal() and len(value) <= 9: + return int(value) + raise _Refused(_BAD_NUMBER) + + +def _text(value: Any, *, required: bool, problem: str) -> str: + if value is None and not required: + return "" + if not isinstance(value, str) or len(value) > MAX_PATTERN_CHARS or (required and not value): + raise _Refused(problem) + return value + + +def _finished_text(lines: list[str], *, stopped: bool, nothing: str) -> str: + body = "\n".join(lines) + if stopped: + return f"{body}\n{STOPPED_EARLY}" if body else STOPPED_EARLY + return body or nothing + + +# Room for the note, so a capped result still ends with it inside the limit. +_ROOM = MAX_RESULT_CHARS - len(STOPPED_EARLY) - 1 + + +def _read_lines(root: str, arguments: dict[str, Any]) -> str: + joined, real = _place(root, arguments.get("path")) + if real == root: + raise _Refused(_NOT_A_FILE) + offset = max(_whole(arguments.get("offset"), 1), 1) + limit = min(max(_whole(arguments.get("limit"), MAX_READ_LINES), 1), MAX_READ_LINES) + _one_file(joined) + lines = _read_bytes(joined).splitlines() + if not lines: + return "(empty file)" + if offset > len(lines): + return f"(the file has {len(lines)} lines; start at a lower offset)" + shown: list[str] = [] + size = 0 + number = offset + for raw in lines[offset - 1 : offset - 1 + limit]: + line = f"{number}\t{raw.decode('utf-8', 'replace')[:MAX_LINE_CHARS]}" + # Room for the continuation line and the note. + if size + len(line) + 1 > _ROOM - 80: + shown.append(f"... call again with offset={number}") + return _finished_text(shown, stopped=True, nothing="") + shown.append(line) + size += len(line) + 1 + number += 1 + remaining = len(lines) - (number - 1) + if remaining > 0: + shown.append(f"... ({remaining} more lines; call again with offset={number})") + return "\n".join(shown) + + +_BRACE = re.compile(r"\{([^{}]*,[^{}]*)\}") +_MAX_FORMS = 32 + + +def _pattern_forms(pattern: str) -> tuple[str, ...]: + """The pattern as a shell reads it: fnmatch has no ``{a,b}``, and its ``**/`` needs a folder + in front, so ``**/package.json`` would miss the one at the root. Never more than 32 forms.""" + forms, done = [pattern], [] + while forms and len(done) + len(forms) <= _MAX_FORMS: + form = forms.pop() + brace = _BRACE.search(form) + if brace is None: + done.append(form) + continue + forms.extend( + form[: brace.start()] + choice + form[brace.end() :] + for choice in brace.group(1).split(",") + ) + done.extend(forms) + grown = [] + for form in done[:_MAX_FORMS]: + grown.append(form) + flat = form.replace("/**/", "/") + while flat.startswith("**/"): + flat = flat[3:] + if flat: + grown.append(flat) + return tuple(dict.fromkeys(grown)) + + +def _name_matches(relative: str, forms: tuple[str, ...]) -> bool: + name = relative.rsplit("/", 1)[-1] + return any( + fnmatch.fnmatchcase(relative, form) or fnmatch.fnmatchcase(name, form) for form in forms + ) + + +def _glob_files(root: str, arguments: dict[str, Any]) -> str: + pattern = _text(arguments.get("pattern"), required=True, problem=_BAD_PATTERN) + _, start = _place(root, arguments.get("path")) + if not os.path.isdir(start): + raise _Refused(_NOT_A_FOLDER if os.path.exists(start) else _MISSING) + forms = _pattern_forms(pattern) + budget = _Budget() + found: list[str] = [] + size = 0 + capped = False + for path, relative in _walk(root, start, budget): + if not _name_matches(os.path.relpath(path, start), forms): + continue + if size + len(relative) + 1 > _ROOM: + capped = True + break + found.append(relative) + size += len(relative) + 1 + return _finished_text(found, stopped=budget.stopped or capped, nothing="No file matches.") + + +def _grep_files(root: str, arguments: dict[str, Any]) -> str: + text = _text(arguments.get("text"), required=True, problem=_BAD_TEXT) + wanted = _text(arguments.get("glob"), required=False, problem=_BAD_GLOB) + ignore_case = arguments.get("ignore_case") + if ignore_case is None: + ignore_case = False + if not isinstance(ignore_case, bool): + raise _Refused(_BAD_IGNORE_CASE) + joined, start = _place(root, arguments.get("path")) + budget = _Budget() + if os.path.isdir(start): + targets: Iterator[tuple[str, str]] = _walk(root, start, budget) + else: + _one_file(joined) + targets = iter([(joined, os.path.relpath(start, root))]) + forms = _pattern_forms(wanted) if wanted else () + needle = text.lower() if ignore_case else text + exact = text.encode("utf-8") + hits: list[str] = [] + size = 0 + capped = False + for path, relative in targets: + if forms and not _name_matches(relative, forms): + continue + try: + if budget.read_bytes + os.lstat(path).st_size > MAX_READ_BYTES: + budget.stopped = True + break + data = _read_bytes(path) + except (OSError, _Refused): + continue + budget.read_bytes += len(data) + for number, raw in enumerate(data.splitlines(), 1): + # A line this long is minified or generated, and matching it is what makes a + # search slow. + if len(raw) > MAX_LINE_BYTES: + continue + if not ignore_case and exact not in raw: + continue + line = raw.decode("utf-8", "replace") + if ignore_case and needle not in line.lower(): + continue + hit = f"{relative}:{number}:{line[:HIT_CHARS]}" + if size + len(hit) + 1 > _ROOM: + capped = True + break + hits.append(hit) + size += len(hit) + 1 + if capped: + break + return _finished_text(hits, stopped=budget.stopped or capped, nothing="No line matches.") + + +# -- the answer -------------------------------------------------------------------------------- + + +def _stripped(path: str, prefixes: tuple[str, ...]) -> str: + for prefix in prefixes: + if path == prefix.rstrip("/") and prefix != "./": + return "" + if path.startswith(prefix): + return path[len(prefix) :] + return path + + +def _checked_pair( + root: str, prefixes: tuple[str, ...], item: Any +) -> tuple[str, str] | None: + if not isinstance(item, dict): + return None + folder, entry = item.get("folder"), item.get("entry") + if not isinstance(folder, str) or not isinstance(entry, str): + return None + folder, entry = _stripped(folder, prefixes), _stripped(entry, prefixes) + if not fits_path_rule(entry): + return None + if folder and (not fits_path_rule(folder) or not entry.startswith(folder + "/")): + return None + if _in_skipped_folder(entry.split("/")[:-1]): + return None + path = os.path.join(root, entry) + try: + if not stat.S_ISREG(os.lstat(path).st_mode): + return None + except OSError: + return None + real_folder = os.path.realpath(os.path.join(root, folder)) if folder else root + if not _under(real_folder, root) or not os.path.isdir(real_folder): + return None + if not _under(os.path.realpath(path), real_folder): + return None + return folder, entry + + +def _uncarried_entry(root: str, prefixes: tuple[str, ...], item: Any) -> str | None: + """A real file the model named, inside the source, whose path breaks only the platform's + rule (a Next.js ``app/(chat)/...`` route, a space). A missing or outside file is the model's + mistake and isn't one.""" + if not isinstance(item, dict) or not isinstance(item.get("entry"), str): + return None + entry = _stripped(item["entry"], prefixes) + if not entry or len(entry) > MAX_PATH_CHARS or fits_path_rule(entry): + return None + parts = entry.split("/") + if len(parts) > MAX_PATH_PARTS or {"", ".", ".."} & set(parts): + return None + if _in_skipped_folder(parts[:-1]): + return None + path = os.path.join(root, entry) + try: + if not stat.S_ISREG(os.lstat(path).st_mode): + return None + except (OSError, ValueError): + return None + if not _under(os.path.realpath(path), root): + return None + return entry + + +def _checked_pairs( + root: str, prefixes: tuple[str, ...], agents: list[Any] +) -> tuple[tuple[tuple[str, str], ...], int, tuple[str, ...]]: + kept: list[tuple[str, str]] = [] + entries: set[str] = set() + uncarried: list[str] = [] + refused = 0 + for item in agents: + pair = _checked_pair(root, prefixes, item) + if pair is None: + refused += 1 + entry = _uncarried_entry(root, prefixes, item) + if entry is not None: + uncarried.append(entry) + elif pair[1] not in entries: + entries.add(pair[1]) + kept.append(pair) + return tuple(kept), refused, tuple(uncarried) + + +def _submit_server(source: Path, answer: _Answer) -> ToolServer: + root = os.path.realpath(str(source)) + prefixes = tuple( + dict.fromkeys((str(source).rstrip("/") + "/", root.rstrip("/") + "/", "./")) + ) + + async def submit_agents(arguments: dict[str, Any]) -> dict[str, Any]: + try: + if answer.pairs: + return _ok(SUBMIT_AFTER_SUBMIT) + agents = arguments.get("agents") + if not isinstance(agents, list) or len(agents) > MAX_PAIRS: + return _error(_BAD_SUBMIT) + kept, refused, uncarried = await _in_thread( + _checked_pairs, root, prefixes, agents + ) + # Another call may have finished while this one was checking. Nothing awaits + # between this check and the set below, so two calls of one reply, run at once, + # can never both keep their pairs. + if answer.pairs: + return _ok(SUBMIT_AFTER_SUBMIT) + answer.called = True + answer.uncarried.update(uncarried) + if kept: + answer.pairs = kept + answer.refused = refused + return _ok( + f"Recorded {len(kept)}; {refused} refused. " + "Your work is done: end your turn now." + ) + if uncarried: + return _ok( + f"Recorded 0; {refused} refused. {len(set(uncarried))} name a file whose " + "path the platform can't carry: a path may hold only letters, digits, '.', " + "'_', '-' and '/'. Those agents can't be listed, so don't submit them " + "again. Submit any others, or end your turn." + ) + return _ok( + f"Recorded 0; {refused} refused. Each entry must be a file inside the " + "repository and inside its folder. Submit again with corrected paths, or end " + "your turn if there are no agents." + ) + except asyncio.TimeoutError: + return _error(_TOO_SLOW) + except Exception: + return _error(_FAILED) + + return ToolServer( + name="scan", + tools=[ + ToolSpec( + name="submit_agents", + description=( + "Submit the agents you found, once: each a folder (\"\" for the root) and " + "the entry file inside it, both relative to the repository's root. An empty " + "list means there are none." + ), + input_schema={ + "type": "object", + "properties": { + "agents": { + "type": "array", + "maxItems": MAX_PAIRS, + "items": { + "type": "object", + "properties": { + "folder": {"type": "string"}, + "entry": {"type": "string"}, + }, + "required": ["folder", "entry"], + }, + } + }, + "required": ["agents"], + }, + handler=submit_agents, + ) + ], + ) diff --git a/src/fi/alk/harness/scan_document.py b/src/fi/alk/harness/scan_document.py new file mode 100644 index 000000000..3ea15d0e2 --- /dev/null +++ b/src/fi/alk/harness/scan_document.py @@ -0,0 +1,666 @@ +"""The agent scan's document, `agents.json`, in the shape the platform's reader accepts. + +The platform treats a file its reader rejects as a failed reading and retries it, and the retry +would fail the same way. So every row is checked against the reader's rules before it is written, +a row that fails is left out rather than failing the scan, and the whole file is checked once more +before it leaves. The rules mirror the reader's serializer at the commit the fixture names; the +fixture test holds the two together. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +import tempfile +import unicodedata +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from .scan_walk import fits_path_rule + +if TYPE_CHECKING: + from .job import HarnessJob + from .scan_walk import Candidate + +logger = logging.getLogger(__name__) + +AGENTS_FILE = "agents.json" +SCHEMA_VERSION = "futureagi.agent-scan.v1" +MAX_AGENTS = 50 +MAX_BYTES = 1_048_576 +ENV_NAMES_KEPT = 50 +NAME_MAX = 255 +EVIDENCE_MAX = 200 +EVIDENCE_KEPT = 5 + +# The reader's words, in its order. The first ten are the frameworks it names. +FRAMEWORK_NAMES = ( + "langchain", + "langgraph", + "livekit", + "claude_agent_sdk", + "openai_agents", + "pydantic_ai", + "google_adk", + "crewai", + "pipecat", + "deepagents", + "other", + "none", + "unknown", +) +_NAMED = frozenset(FRAMEWORK_NAMES[:10]) +_STATUSES = ("found", "none", "error") +_CONFIDENCES = ("high", "medium", "low") +_CONFIDENCE_RANK = {word: rank for rank, word in enumerate(_CONFIDENCES)} + +# The reader's patterns, as written there. It trims a value before matching, and a trimmed value +# has no trailing line break, so `fullmatch` here gives the same verdicts as its `search`. +_SUBFOLDER_PATTERN = re.compile( + r"^(?!/)(?!.*(^|/)\.{1,2}(/|$))[A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*$" +) +_ENV_NAME_PATTERN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_COMMIT_PATTERN = re.compile(r"^[0-9a-f]{40}$") +_CODE_PATTERN = re.compile(r"^[a-z0-9_]+$") + +_TOP_KEYS = ("schema_version", "status", "commit_sha", "agents", "truncated", "error") +_AGENT_KEYS = ( + "name", + "subfolder", + "entry", + "framework", + "confidence", + "evidence", + "env_names", + "estimates", +) +_FRAMEWORK_KEYS = ("name", "version", "label") +_ESTIMATE_KEYS = ("tools", "prompt_files") +_ERROR_KEYS = ("code", "message") + +# Removed from names: control, format, surrogate and private-use characters. None of them shows +# as text, and a surrogate can't be written as UTF-8 at all. +_HIDDEN_CATEGORIES = frozenset({"Cc", "Cf", "Cs", "Co"}) + +# A sentence's one free part is a path, shortened to the room its template leaves. Every +# template leaves at least this much, so a path always shows something more than its file name. +_PATH_ROOM = 120 +_TEMPLATES = { + "builds": "Builds with {detail} in {where}.", + "graph": "Its langgraph.json lists a graph: {where}.", + "agent_py": "An agent.py using a framework or a model provider: {where}.", + # The provider's name follows "for", so no article has to agree with it. + "client": "Creates a client for {detail} in {where}.", + "model": "Calls a chat model through {detail} in {where}.", + "project": "Its project file is {where}.", + "openai_agents_set_aside": ( + "Uses the OpenAI Agents SDK, set aside because a folder named agents/ shadows it." + ), + "shares_folder": ( + "Another agent listed here shares this folder, so a build must name its entry." + ), + "graphs_may_stop": ( + "This folder's langgraph.json lists more than one graph, so a build may stop." + ), + "agent_py_may_stop": "More than one agent.py sits under this folder, so a build may stop.", + "not_finished": "The framework check for this folder did not finish.", + "partial": "Not every file under this folder was read in time, so counts may be short.", + "found_by_ai": "Found by the AI step, not by the code check.", +} +_LONGEST_TEMPLATE = max(len(template) for template in _TEMPLATES.values()) +_UNNAMED_PATH = "a file in this folder" +_DETAIL = re.compile(r"[A-Za-z0-9_.-]{1,40}") +_DETAIL_FALLBACK = { + "builds": "an agent framework", + "client": "a model provider", + "model": "an agent framework", +} +_FRAMEWORK_TITLES = { + "langchain": "LangChain", + "langgraph": "LangGraph", + "livekit": "LiveKit Agents", + "claude_agent_sdk": "the Claude Agent SDK", + "openai_agents": "the OpenAI Agents SDK", + "pydantic_ai": "Pydantic AI", + "google_adk": "Google ADK", + "crewai": "CrewAI", + "pipecat": "Pipecat", + "deepagents": "Deep Agents", +} +# Line numbers past this are a broken fact, and would eat the room the path needs. +_LINE_SHOWN_BELOW = 10_000_000 + + +@dataclass(frozen=True) +class Row: + candidate: Candidate + framework: dict[str, Any] + finished: bool + found_by_ai: bool + + +class ScanResultInvalid(Exception): + """The document as a whole breaks the platform's rules, so writing it would only be retried.""" + + def __init__(self, problem: str) -> None: + super().__init__( + f"The agent list breaks the platform's rules at {problem}; " + "this is a fault in the scan, not in the repository." + ) + self.problem = problem + + +def _clean_name(text: str) -> str: + """``text`` without hidden characters, its runs of any kind of space made one space.""" + # Spaces first: a tab or line break is itself a control character, and dropping it before + # splitting would join the words on either side of it. + words = ( + "".join(ch for ch in word if unicodedata.category(ch) not in _HIDDEN_CATEGORIES) + for word in text.split() + ) + return " ".join(word for word in words if word) + + +def root_name(job: HarnessJob) -> str: + """The name for an agent at the repository's root.""" + choices: list[str] = [] + named = job.metadata.get("agent_name") + if isinstance(named, str): + choices.append(named) + repository = job.source.repository + if isinstance(repository, str): + choices.append(repository.rstrip("/").rsplit("/", 1)[-1].removesuffix(".git")) + for choice in choices: + cleaned = _clean_name(choice) + if cleaned: + return cleaned + return "agent" + + +def _base_name(subfolder: str, root: str) -> str: + last = subfolder.rsplit("/", 1)[-1] if subfolder else "" + for choice in (last, root): + cleaned = _clean_name(choice) + if cleaned: + return cleaned + return "agent" + + +def _names(rows: Sequence[Row], root: str) -> list[str]: + bases = [_base_name(row.candidate.subfolder, root)[:NAME_MAX] for row in rows] + names = list(bases) + for suffix_of in (lambda entry: entry.rsplit("/", 1)[-1], lambda entry: entry): + counts = Counter(names) + names = [ + _with_suffix(base, f" ({suffix_of(row.candidate.entry)})") + if counts[name] > 1 + else name + for base, name, row in zip(bases, names, rows) + ] + return names + + +def _with_suffix(base: str, suffix: str) -> str: + # The suffix is what tells two agents apart, so the base gives way to it, and a suffix too + # long on its own keeps its end, where the entry's file name is. + if len(suffix) >= NAME_MAX: + return suffix[-NAME_MAX:] + return base[: NAME_MAX - len(suffix)] + suffix + + +def _shown_path(path: Any, room: int) -> str: + if not isinstance(path, str) or not fits_path_rule(path): + return _UNNAMED_PATH + if len(path) <= room: + return path + parts = path.split("/") + for start in range(1, len(parts)): + tail = "…/" + "/".join(parts[start:]) + if len(tail) <= room: + return tail + return parts[-1][: max(room - 1, 0)] + "…" + + +def _detail(kind: str, detail: str) -> str: + if kind in ("builds", "model") and detail in _FRAMEWORK_TITLES: + return _FRAMEWORK_TITLES[detail] + if _DETAIL.fullmatch(detail): + return detail + return _DETAIL_FALLBACK.get(kind, "") + + +def _sentence(kind: str, path: Any = "", line: Any = 0) -> str | None: + """The fixed sentence for one fact, or ``None`` for a kind with no sentence. + + A path appears only when the platform's path rule accepts it, so no file name that rule + refuses, and no character that can't be written, ever reaches the document. + """ + head, _, detail = kind.partition(" ") + template = _TEMPLATES.get(head) + if template is None: + return None + words = {"detail": _detail(head, detail), "where": ""} + if "{where}" in template: + at_line = ( + f", line {line}" + if isinstance(line, int) and not isinstance(line, bool) and 0 < line < _LINE_SHOWN_BELOW + else "" + ) + fixed = len(template.format(**words)) + len(at_line) + words["where"] = _shown_path(path, EVIDENCE_MAX - fixed) + at_line + sentence = template.format(**words) + return sentence if len(sentence) <= EVIDENCE_MAX else sentence[: EVIDENCE_MAX - 1] + "…" + + +def _framework_name(row: Row) -> str: + name = row.framework.get("name") if isinstance(row.framework, Mapping) else None + return name if isinstance(name, str) else "" + + +def _confidence(row: Row, shares: bool) -> str: + candidate = row.candidate + name = _framework_name(row) + if ( + name in ("unknown", "none") + or row.found_by_ai + or shares + or candidate.build_may_stop + or candidate.partial + or not row.finished + # A chat model alone may be one step of something larger, not an agent. + or candidate.sign == "model" + ): + return "low" + if name in _NAMED and name in candidate.builds: + return "high" + return "medium" + + +def _order_key(row: Row, shares: bool) -> tuple[bool, int, str, str]: + candidate = row.candidate + return ( + candidate.side, + _CONFIDENCE_RANK[_confidence(row, shares)], + candidate.subfolder, + candidate.entry, + ) + + +def _choice_key(row: Row, models_first: bool) -> tuple[bool, bool, int, str, str]: + side, rank, subfolder, entry = _order_key(row, False) + behind = ( + models_first + and not side + and not row.found_by_ai + and row.candidate.sign in ("client", "model") + ) + return (side, behind, rank, subfolder, entry) + + +def _evidence(row: Row, shares: bool) -> list[str]: + candidate = row.candidate + set_aside_shown = _framework_name(row) != "openai_agents" + facts = [ + _sentence(kind, path, line) + for kind, path, line in candidate.evidence + if kind != "openai_agents_set_aside" or set_aside_shown + ] + caveats = [ + _sentence(kind) + for kind, applies in ( + ("found_by_ai", row.found_by_ai), + ("shares_folder", shares), + ("graphs_may_stop", candidate.build_may_stop == "graphs"), + ("agent_py_may_stop", candidate.build_may_stop == "agent_py"), + ("not_finished", not row.finished), + ("partial", candidate.partial), + ) + if applies + ] + # The strongest fact leads; the caveats that explain a low confidence come before the rest. + known = [sentence for sentence in facts if sentence] + ordered = known[:1] + [sentence for sentence in caveats if sentence] + known[1:] + return list(dict.fromkeys(ordered))[:EVIDENCE_KEPT] + + +def _wire_row(row: Row, *, name: str, shares: bool) -> dict[str, Any]: + candidate = row.candidate + framework = row.framework + return { + "name": name, + "subfolder": candidate.subfolder, + "entry": candidate.entry, + "framework": dict(framework) if isinstance(framework, Mapping) else framework, + "confidence": _confidence(row, shares), + "evidence": _evidence(row, shares), + "env_names": list(candidate.env_names[:ENV_NAMES_KEPT]), + "estimates": {"tools": candidate.tools, "prompt_files": candidate.prompt_files}, + } + + +def _document( + *, + status: str, + commit_sha: str | None, + agents: list[dict[str, Any]], + truncated: bool, + error: dict[str, Any] | None, +) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "status": status, + "commit_sha": commit_sha, + "agents": agents, + "truncated": truncated, + "error": error, + } + + +def _commit(commit_sha: str | None) -> str | None: + if not isinstance(commit_sha, str): + return None + lowered = commit_sha.lower() + return lowered if _COMMIT_PATTERN.fullmatch(lowered) else None + + +def choose_rows( + rows: Sequence[Row], *, limit: int = MAX_AGENTS, models_first: bool = False +) -> tuple[list[Row], bool]: + """The rows to write, one per entry and at most ``limit``, and whether any were left out. + + Who shares a folder isn't known until the rows are chosen, so the order here treats every + row as alone in its folder. With ``models_first``, the model's rows outside side folders + are chosen before the walk's rows that only call a model: the model was paid to look past + those, so what it found is the likelier agent, and it must not be the row the cap drops. + """ + seen: set[str] = set() + unique: list[Row] = [] + for row in rows: + if row.candidate.entry in seen: + continue + seen.add(row.candidate.entry) + unique.append(row) + unique.sort(key=lambda row: _choice_key(row, models_first)) + chosen = unique[: max(limit, 0)] + return chosen, len(chosen) != len(rows) + + +def finish( + *, + rows: Sequence[Row], + root_name: str, + commit_sha: str | None, + truncated: bool, + error: tuple[str, str] | None, + models_first: bool = False, +) -> tuple[bytes, bool]: + """The bytes of ``agents.json``, and whether a row was left out by the cap or the self-check. + + Rows given with an error win: what was found is worth more than why the scan stopped early, + and ``truncated`` says it stopped. ``models_first`` changes which rows are chosen, never + the order they are written in. + """ + root = _clean_name(root_name) or "agent" + commit = _commit(commit_sha) + chosen, dropped = choose_rows(rows, limit=MAX_AGENTS, models_first=models_first) + + kept: list[Row] = [] + for row in chosen: + name = _base_name(row.candidate.subfolder, root)[:NAME_MAX] + alone = _wire_row(row, name=name, shares=False) + problems = wire_problems( + _document(status="found", commit_sha=None, agents=[alone], truncated=False, error=None) + ) + if problems: + logger.warning("agent scan: row left out, problem=%s", problems[0]) + dropped = True + continue + kept.append(row) + + folders = Counter(row.candidate.subfolder for row in kept) + shares = [folders[row.candidate.subfolder] > 1 for row in kept] + order = sorted(range(len(kept)), key=lambda index: _order_key(kept[index], shares[index])) + kept = [kept[index] for index in order] + shares = [shares[index] for index in order] + names = _names(kept, root) + agents = [ + _wire_row(row, name=name, shares=shared) for row, name, shared in zip(kept, names, shares) + ] + + if rows and error is not None: + logger.info("agent scan: error dropped for rows, code=%s", error[0]) + if agents: + status, wire_error = "found", None + elif error is not None and not rows: + status, wire_error = "error", {"code": error[0], "message": error[1]} + else: + status, wire_error = "none", None + document = _document( + status=status, + commit_sha=commit, + agents=agents, + truncated=bool(truncated or dropped or (rows and error is not None)), + error=wire_error, + ) + problems = wire_problems(document) + if problems: + raise ScanResultInvalid(problems[0]) + body = json.dumps(document, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + if len(body) > MAX_BYTES: + raise ScanResultInvalid("document.too_large") + logger.info("agent scan: rows written=%d left out=%d", len(agents), len(rows) - len(agents)) + return body, dropped + + +def _at(where: str, key: str) -> str: + return f"{where}.{key}" if where else key + + +def _keys_problems(value: Any, where: str, keys: tuple[str, ...]) -> list[str]: + if not isinstance(value, dict): + return [_at(where, "not_an_object")] + problems = [_at(_at(where, key), "missing") for key in keys if key not in value] + if set(value) - set(keys): + problems.append(_at(where, "unknown_keys")) + return problems + + +def _text_problems( + value: Any, + where: str, + *, + longest: int, + nullable: bool = False, + blank: bool = False, + pattern: re.Pattern[str] | None = None, +) -> list[str]: + if value is None: + return [] if nullable else [f"{where}.null"] + if not isinstance(value, str): + return [f"{where}.type"] + # The reader trims before every check, so lengths and patterns are judged on the trimmed text. + text = value.strip() + if not text: + return [] if blank else [f"{where}.blank"] + problems = [] + if len(text) > longest: + problems.append(f"{where}.too_long") + if "\x00" in text: + problems.append(f"{where}.null_byte") + if any(0xD800 <= ord(ch) <= 0xDFFF for ch in text): + problems.append(f"{where}.surrogate") + if pattern is not None and not pattern.fullmatch(text): + problems.append(f"{where}.pattern") + return problems + + +def _choice_problems(value: Any, where: str, choices: tuple[str, ...]) -> list[str]: + if value is None: + return [f"{where}.null"] + if not isinstance(value, str) or value not in choices: + return [f"{where}.choice"] + return [] + + +def _count_problems(value: Any, where: str) -> list[str]: + if not isinstance(value, int) or isinstance(value, bool): + return [f"{where}.type"] + return [f"{where}.negative"] if value < 0 else [] + + +def _texts_problems( + value: Any, where: str, *, most: int, longest: int, pattern: re.Pattern[str] | None = None +) -> list[str]: + if not isinstance(value, list): + return [f"{where}.not_a_list"] + problems = [f"{where}.too_many"] if len(value) > most else [] + for index, item in enumerate(value): + problems += _text_problems(item, f"{where}[{index}]", longest=longest, pattern=pattern) + return problems + + +def _framework_problems(value: Any, where: str) -> list[str]: + problems = _keys_problems(value, where, _FRAMEWORK_KEYS) + if not isinstance(value, dict): + return problems + if "name" in value: + problems += _choice_problems(value["name"], f"{where}.name", FRAMEWORK_NAMES) + if "version" in value: + problems += _text_problems(value["version"], f"{where}.version", longest=64, nullable=True) + if "label" in value: + problems += _text_problems(value["label"], f"{where}.label", longest=128, nullable=True) + if not problems and (value["name"] == "other") != (value["label"] is not None): + problems.append(f"{where}.label.other_only") + return problems + + +def _agent_problems(agent: Any, where: str) -> list[str]: + problems = _keys_problems(agent, where, _AGENT_KEYS) + if not isinstance(agent, dict): + return problems + if "name" in agent: + problems += _text_problems(agent["name"], f"{where}.name", longest=NAME_MAX) + for key in ("subfolder", "entry"): + if key in agent: + problems += _text_problems( + agent[key], + f"{where}.{key}", + longest=512, + blank=key == "subfolder", + pattern=_SUBFOLDER_PATTERN, + ) + if "framework" in agent: + problems += _framework_problems(agent["framework"], f"{where}.framework") + if "confidence" in agent: + problems += _choice_problems(agent["confidence"], f"{where}.confidence", _CONFIDENCES) + if "evidence" in agent: + problems += _texts_problems( + agent["evidence"], f"{where}.evidence", most=EVIDENCE_KEPT, longest=EVIDENCE_MAX + ) + if "env_names" in agent: + problems += _texts_problems( + agent["env_names"], + f"{where}.env_names", + most=200, + longest=128, + pattern=_ENV_NAME_PATTERN, + ) + if "estimates" in agent: + estimates = agent["estimates"] + problems += _keys_problems(estimates, f"{where}.estimates", _ESTIMATE_KEYS) + if isinstance(estimates, dict): + for key in _ESTIMATE_KEYS: + if key in estimates: + problems += _count_problems(estimates[key], f"{where}.estimates.{key}") + return problems + + +def _error_problems(error: Any) -> list[str]: + if error is None: + return [] + problems = _keys_problems(error, "error", _ERROR_KEYS) + if not isinstance(error, dict): + return problems + if "code" in error: + problems += _text_problems(error["code"], "error.code", longest=64, pattern=_CODE_PATTERN) + if "message" in error: + problems += _text_problems(error["message"], "error.message", longest=2000) + return problems + + +def wire_problems(document: dict[str, Any]) -> list[str]: + """Where ``document`` breaks the platform reader's rules, as problem names, never values. + + An empty list means the reader accepts it. Where the reader is looser than the written + contract (it ignores unknown keys, and turns numbers into text and ``"true"`` into a boolean), + this is stricter: the kit never needs that leniency, and relying on it would tie the file to + the reader's version. + """ + problems = _keys_problems(document, "", _TOP_KEYS) + if not isinstance(document, dict): + return problems + if "schema_version" in document: + problems += _choice_problems( + document["schema_version"], "schema_version", (SCHEMA_VERSION,) + ) + if "status" in document: + problems += _choice_problems(document["status"], "status", _STATUSES) + if "commit_sha" in document: + problems += _text_problems( + document["commit_sha"], + "commit_sha", + longest=40, + nullable=True, + pattern=_COMMIT_PATTERN, + ) + agents = document.get("agents") + if "agents" in document: + if not isinstance(agents, list): + problems.append("agents.not_a_list") + elif len(agents) > MAX_AGENTS: + problems.append("agents.too_many") + else: + for index, agent in enumerate(agents): + problems += _agent_problems(agent, f"agents[{index}]") + if "truncated" in document and not isinstance(document["truncated"], bool): + problems.append("truncated.type") + if "error" in document: + problems += _error_problems(document["error"]) + if not problems: + status, error = document["status"], document["error"] + if status == "found" and (not agents or error is not None): + problems.append("status.found_needs_agents_and_no_error") + elif status == "none" and (agents or error is not None): + problems.append("status.none_needs_no_agents_and_no_error") + elif status == "error" and (agents or error is None): + problems.append("status.error_needs_an_error_and_no_agents") + return problems + + +def write_agents_json(output: Path, body: bytes) -> Path: + """Write ``body`` as ``output/agents.json`` whole, so a reader never sees half a file.""" + output = Path(output) + output.mkdir(parents=True, exist_ok=True) + path = output / AGENTS_FILE + temporary = tempfile.NamedTemporaryFile( + "wb", dir=output, prefix=".agents-", suffix=".tmp", delete=False + ) + try: + with temporary: + temporary.write(body) + temporary.flush() + os.fsync(temporary.fileno()) + os.chmod(temporary.name, 0o644) + os.replace(temporary.name, path) + except BaseException: + try: + os.unlink(temporary.name) + except OSError: + pass + raise + return path diff --git a/src/fi/alk/harness/scan_folders.py b/src/fi/alk/harness/scan_folders.py new file mode 100644 index 000000000..5dd781f69 --- /dev/null +++ b/src/fi/alk/harness/scan_folders.py @@ -0,0 +1,213 @@ +"""Asks TH-8805's framework check about one agent folder at a time, in a child the scan can stop. + +`detect_framework` bounds its work, not its time, so one slow folder could hold the scan past +its clock. In a child process a check can be abandoned at the deadline by killing it, and a +child the system kills costs that folder's word rather than the whole scan. +""" + +from __future__ import annotations + +import logging +import multiprocessing +import os +import time +from collections.abc import Callable +from dataclasses import dataclass +from multiprocessing.connection import Connection +from multiprocessing.process import BaseProcess +from pathlib import Path +from typing import Any + +from .framework_detection import detect_framework +from .source_folder import agent_folder + +logger = logging.getLogger(__name__) + +# Loaded once in the fork server, so a restarted child doesn't import the kit again. +_PRELOAD = ["fi.alk.harness.framework_detection"] +# A killed child is gone almost at once; the bound only stops a wedged reap holding the scan. +_JOIN_SECONDS = 5.0 + + +@dataclass(frozen=True) +class FolderWord: + framework: dict[str, Any] + finished: bool + + +class FolderCheckerBroken(Exception): + """The child couldn't start, or ended without an answer other than by a signal.""" + + +class ClockRanOut(Exception): + """The scan's deadline passed before a folder's answer came back.""" + + +def _unknown() -> dict[str, Any]: + return {"name": "unknown", "version": None, "label": None} + + +def _answer(source: Path, subfolder: str) -> dict[str, Any]: + try: + # The build's own call for a named folder. Nothing is installed during a scan, so + # there is no package folder to read. + return detect_framework( + agent_folder(source, subfolder) if subfolder else source, + repo_root=source, + package_dir=None, + ) + except Exception: + # A folder the build would refuse has no word; it is not a reason to end the child. + return {"name": "unknown", "version": None} + + +def _serve(conn: Connection) -> None: + """The child's loop: one answer per request, until the parent's end closes.""" + while True: + try: + source, subfolder = conn.recv() + conn.send(_answer(Path(source), subfolder)) + except (EOFError, OSError): + return + + +def _framework(answer: Any) -> dict[str, Any]: + name = answer.get("name") if isinstance(answer, dict) else None + if not isinstance(name, str): + return _unknown() + label = answer.get("label") if name == "other" else None + return { + "name": name, + "version": None, + "label": label if isinstance(label, str) else None, + } + + +class FolderChecker: + """One child process, started on the first check, answering for one folder at a time. + + Answers are kept per folder for the life of the checker, since agents that share a folder + get the same word and a folder whose check killed one child would likely kill the next. + """ + + pid: int | None + + def __init__(self) -> None: + self.pid = None + self._process: BaseProcess | None = None + self._conn: Connection | None = None + self._words: dict[tuple[str, str], FolderWord] = {} + + def __enter__(self) -> FolderChecker: + return self + + def __exit__(self, *exc: object) -> None: + self._end_child() + + def check( + self, + source: Path, + subfolder: str, + *, + deadline: float, + clock: Callable[[], float] = time.monotonic, + ) -> FolderWord: + if deadline - clock() <= 0: + raise ClockRanOut("The scan's clock ran out before this folder was checked.") + key = (os.fspath(source), subfolder) + known = self._words.get(key) + if known is not None: + return FolderWord(dict(known.framework), known.finished) + if self._process is None: + self._start() + conn = self._conn + assert conn is not None + try: + conn.send((source, subfolder)) + except OSError: + return self._keep(key, self._ended()) + # A deadline that passed while the child started still ends it: a late answer isn't taken. + if not conn.poll(max(deadline - clock(), 0.0)): + self._end_child() + logger.warning("folder checker: the clock ran out, child ended") + raise ClockRanOut("The scan's clock ran out while a folder was being checked.") + try: + answer = conn.recv() + except (EOFError, OSError): + return self._keep(key, self._ended()) + return self._keep(key, FolderWord(_framework(answer), finished=True)) + + def _keep(self, key: tuple[str, str], word: FolderWord) -> FolderWord: + self._words[key] = word + return FolderWord(dict(word.framework), word.finished) + + def _start(self) -> None: + parent: Connection | None = None + child: Connection | None = None + try: + context = multiprocessing.get_context("forkserver") + context.set_forkserver_preload(_PRELOAD) + parent, child = context.Pipe() + process = context.Process(target=_serve, args=(child,), daemon=True) + process.start() + except Exception as error: + if parent is not None: + parent.close() + logger.warning( + "folder checker: the child could not start, class=%s", + type(error).__name__, + ) + raise FolderCheckerBroken( + "The folder checker's child process could not start." + ) from error + finally: + # The child has its own copy. Closing ours lets a dead child read as end of file. + if child is not None: + child.close() + self._process, self._conn, self.pid = process, parent, process.pid + + def _ended(self) -> FolderWord: + """The child stopped answering: a signal costs this folder's word, anything else is broken.""" + process = self._process + assert process is not None + code: int | None = None + try: + # The exit code is only known once the child is reaped. + process.join(_JOIN_SECONDS) + code = process.exitcode + except Exception: + code = None + self._end_child() + if code is not None and code < 0: + logger.warning("folder checker: child ended by signal %d", -code) + return FolderWord(_unknown(), finished=False) + logger.warning("folder checker: child ended without an answer, exit code %s", code) + if code is None: + raise FolderCheckerBroken( + "The folder checker's child process stopped answering and was ended." + ) + raise FolderCheckerBroken( + "The folder checker's child process ended without an answer, " + f"with exit code {code}." + ) + + def _end_child(self) -> None: + """Kills, reaps and forgets the child, if there is one. Never raises.""" + process, conn = self._process, self._conn + self._process = self._conn = None + self.pid = None + if process is not None: + try: + process.kill() + process.join(_JOIN_SECONDS) + process.close() + except Exception as error: + logger.warning( + "folder checker: the child could not be ended, class=%s", + type(error).__name__, + ) + if conn is not None: + try: + conn.close() + except Exception: + pass diff --git a/src/fi/alk/harness/scan_walk.py b/src/fi/alk/harness/scan_walk.py new file mode 100644 index 000000000..c5bdebfbf --- /dev/null +++ b/src/fi/alk/harness/scan_walk.py @@ -0,0 +1,1382 @@ +"""Finds the files in a repository that look like agents, from its files alone. + +One walk reads each file once and runs nothing. Python files go through the framework +check's own per-file reader, so "builds an agent" means what it means to a build, and a +token pass of the scan's own adds what that reader doesn't keep: the repository files a +file imports, provider clients, setting names and tool registrations. + +A candidate imported by another, directly or through files that aren't candidates, is +part of the importer's agent when the importer's sign is as strong or stronger, unless +the importer sits in a side folder and the imported file doesn't. Candidates that import +each other are one agent. +""" + +from __future__ import annotations + +import ast +import codecs +import json +import logging +import math +import os +import posixpath +import re +import time +import tokenize +import warnings +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Iterator, Literal + +from .framework_detection import ( + _ALWAYS_SKIPPED, + _TOP_ONLY_SKIPS, + _FileScan, + _has_long_line, + _is_side_file, + _is_side_part, + _OverBudget, + _own_modules, + _read_file, + _scan_file, + _visible_tokens, +) +from .source_folder import _FOLDER_FORM, _FOLDER_MAX + +logger = logging.getLogger(__name__) + +Sign = Literal["framework", "graph", "agent_py", "client", "model"] +Hazard = Literal["", "graphs", "agent_py"] + +_FILE_BYTES = 1_048_576 +# A hundredth of a folder check's allowance: the crafted files that use a whole one +# take seconds, so one file now costs about a second at most. +_FILE_WORK = 1_000_000_000 +_MAX_FILES = 20_000 +_MAX_BYTES = 268_435_456 +_NAMES_PER_FILE = 200 +_IMPORT_DEPTH = 10 +_MAX_AGENTS = 50 +_DESCRIBE_FILES = 2_000 +_DESCRIBE_BYTES = 33_554_432 +_ENV_NAME_MAX = 128 +_MAX_PARTS = 64 +# The prompt count is an estimate, and every file walked is listed, read or not: past this +# many a tree of prompt files costs memory and adds nothing a person would act on. +_MAX_PROMPT_PATHS = 20_000 +# How many folder entries one look for a build's hazard visits. The hazard is advice, so a +# tree too big to look through gives none rather than the walk's time. +_HAZARD_ENTRIES = 200_000 +# No file system the scan runs on has a path this long, so no file can match. +_MODULE_PATH_MAX = 4_096 + +_STRENGTH = {"framework": 3, "graph": 2, "agent_py": 1, "client": 0, "model": 0} +# A file that only talks to a model: listed unless a stronger agent shares its project. +_WEAKEST = frozenset({"client", "model"}) +_PROJECT_FILES = ("pyproject.toml", "requirements.txt", "setup.py") +_PACKAGE_FILE = "package.json" +_TEMPLATES = (".env.example", ".env.sample", ".env.template") +_LANGGRAPH = "langgraph.json" +_SCRIPT_SUFFIXES = (".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs") +_SOURCE_SUFFIXES = (".py", ".ipynb", *_SCRIPT_SUFFIXES) +# Script files nobody writes an agent in: build output, vendored copies, type stubs and tool +# settings. A Python helper beside only these is no reason to pay for the model. +_BUILT_FOLDERS = frozenset({"dist", "build", "static", "vendor", "node_modules"}) +_BUILT_ENDINGS = (".min.js", ".d.ts", ".d.mts", ".d.cts") +_CONFIG_ENDINGS = tuple(".config" + suffix for suffix in _SCRIPT_SUFFIXES) +# What a build's fallback ignores when it counts `agent.py`. A copy, because the build's +# module is heavy to import; a test holds the two equal. +_BUILD_IGNORED = frozenset({".git", ".venv", "__pycache__", "node_modules"}) + +_CLIENTS = { + ("OpenAI",): "openai", + ("AsyncOpenAI",): "openai", + ("AzureOpenAI",): "openai", + ("Anthropic",): "anthropic", + ("AsyncAnthropic",): "anthropic", + ("genai", "Client"): "google_genai", + ("Mistral",): "mistral", + ("cohere", "Client"): "cohere", + ("cohere", "ClientV2"): "cohere", + ("Groq",): "groq", + ("litellm", "completion"): "litellm", + ("litellm", "acompletion"): "litellm", +} +_PROVIDER_MODULES = ( + ("openai",), + ("anthropic",), + ("google", "genai"), + ("mistralai",), + ("cohere",), + ("groq",), + ("litellm",), +) +_TOOL_DECORATORS = frozenset({"tool", "function_tool"}) +_SETTING_CALLS = (["os", "getenv"], ["os", "environ", "get"]) +_STRING_PREFIXES = frozenset({"", "r", "u"}) + +_ENV_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +# Only the name is captured: the rest of the line holds a value and is never kept. A second +# `=` straight after the first is base64 padding, the last line of a pasted key, not a setting. +_TEMPLATE_LINE = re.compile(r"[ \t]*(?:export[ \t]+)?([^\s=]+)[ \t]*=(?!=)") +_ENCODED_LENGTH = 16 +# An unquoted value this long in base64's alphabets is a pasted key, which may wrap onto the +# lines below it. +_ENCODED_VALUE = re.compile(r"[A-Za-z0-9+/_-]{40,}={0,2}") +_KEY_BEGIN = "-----BEGIN" +_KEY_END = "-----END" +_DOUBLE_QUOTE_CLOSE = re.compile(r'(? None: + self._spent = 0 + + def charge(self, amount: int) -> None: + self._spent += amount + if self._spent > _FILE_WORK: + raise _OverBudget() + + +def fits_path_rule(path: str) -> bool: + """Whether the platform can carry a non-empty path. The root, ``""``, isn't one.""" + return ( + bool(path) + and len(path) <= _FOLDER_MAX + and _FOLDER_FORM.fullmatch(path) is not None + ) + + +def _good_env_name(name: str) -> bool: + return len(name) <= _ENV_NAME_MAX and _ENV_NAME.fullmatch(name) is not None + + +def _join(*parts: str) -> str: + return "/".join(part for part in parts if part) + + +def _under(path: str, folder: str) -> bool: + return not folder or path.startswith(folder + "/") + + +def _below(path: str, folder: str) -> str: + return path[len(folder) + 1 :] if folder else path + + +def _is_side_path(path: str) -> bool: + parts = path.split("/") + return any(_is_side_part(part) for part in parts[:-1]) or _is_side_file(parts[-1]) + + +def _is_written_script(path: str) -> bool: + """A TypeScript or JavaScript file someone wrote, so one an agent may be written in.""" + parts = path.split("/") + name = parts[-1] + return ( + name.endswith(_SCRIPT_SUFFIXES) + and not any(part in _BUILT_FOLDERS for part in parts[:-1]) + and not name.endswith(_BUILT_ENDINGS) + and not name.endswith(_CONFIG_ENDINGS) + ) + + +def _has_link(top: str, folder: str) -> bool: + """Whether reaching ``folder`` passes through a link, or it isn't a real folder.""" + current = top + for part in folder.split("/") if folder else (): + current = os.path.join(current, part) + if os.path.islink(current): + return True + return not os.path.isdir(current) + + +def _folders_to_enter(directory: str, names: list[str], *, at_top: bool) -> list[str]: + return [ + name + for name in names + if name not in _ALWAYS_SKIPPED + and not (at_top and name in _TOP_ONLY_SKIPS) + and not os.path.islink(os.path.join(directory, name)) + ] + + +def _chain( + token: tokenize.TokenInfo | None, tokens: Iterator[tokenize.TokenInfo] +) -> tuple[list[str], tokenize.TokenInfo | None]: + """Reads `n(.n)*` from `token` on; returns its parts and the token after it.""" + parts: list[str] = [] + while token is not None and token.type == tokenize.NAME: + if len(parts) < _MAX_PARTS: + parts.append(token.string) + token = next(tokens, None) + if token is None or token.string != ".": + break + token = next(tokens, None) + return parts, token + + +def _string_value(text: str) -> str | None: + """The value of one plain string literal; None for an f-string, bytes, or anything + too long to be a setting's name.""" + if len(text) > _ENV_NAME_MAX + 8: + return None + body = text.lstrip("rRuUbBfFtT") + if text[: len(text) - len(body)].lower() not in _STRING_PREFIXES: + return None + try: + value = ast.literal_eval(text) + except (ValueError, SyntaxError): + return None + return value if isinstance(value, str) else None + + +class _OwnReader: + """The scan's pass over one file's tokens, for what the framework reader drops.""" + + def __init__(self) -> None: + self.imports: dict[tuple[int, tuple[str, ...]], None] = {} + self.provider = False + self.clients: dict[str, int] = {} + self.env: set[str] = set() + self.tools = 0 + + def read(self, tokens: Iterator[tokenize.TokenInfo]) -> None: + prev = "" + at_start = True + token = next(tokens, None) + while token is not None: + text = token.string + if token.type == tokenize.NAME and prev != ".": + if text == "import": + token, prev, at_start = self._plain_import(tokens), "", False + continue + if text == "from": + token, prev, at_start = self._from_import(tokens), "", False + continue + if prev not in ("def", "class"): + line = token.start[0] + parts, token = _chain(token, tokens) + token = self._call(parts, token, tokens, line) + prev, at_start = parts[-1], False + continue + if text == "@" and at_start: + parts, token = _chain(next(tokens, None), tokens) + if parts and parts[-1] in _TOOL_DECORATORS: + self.tools = min(self.tools + 1, _NAMES_PER_FILE) + prev, at_start = (parts[-1] if parts else "@"), False + continue + if token.type == tokenize.NEWLINE: + prev, at_start = "", True + elif token.type not in (tokenize.INDENT, tokenize.DEDENT): + prev, at_start = text, text == ";" + token = next(tokens, None) + + def _call( + self, + parts: list[str], + token: tokenize.TokenInfo | None, + tokens: Iterator[tokenize.TokenInfo], + line: int, + ) -> tokenize.TokenInfo | None: + following = token.string if token is not None else "" + if following == "(": + sdk = _CLIENTS.get(tuple(parts[-2:])) or _CLIENTS.get(tuple(parts[-1:])) + if sdk is not None and len(self.clients) < _NAMES_PER_FILE: + self.clients.setdefault(sdk, line) + if parts in _SETTING_CALLS: + return self._setting(tokens, (",", ")")) + elif following == "[" and parts == ["os", "environ"]: + return self._setting(tokens, ("]",)) + return token + + def _setting( + self, tokens: Iterator[tokenize.TokenInfo], closers: tuple[str, ...] + ) -> tokenize.TokenInfo | None: + # Called just past the bracket. Only a lone literal is a name: what follows a + # comma is a default, which may be a real value, so it is never looked at. + literal = next(tokens, None) + if literal is None or literal.type != tokenize.STRING: + return literal + after = next(tokens, None) + if after is not None and after.string in closers: + name = _string_value(literal.string) + if ( + name is not None + and _good_env_name(name) + and len(self.env) < _NAMES_PER_FILE + ): + self.env.add(name) + return after + + def _plain_import( + self, tokens: Iterator[tokenize.TokenInfo] + ) -> tokenize.TokenInfo | None: + while True: + parts, token = _chain(next(tokens, None), tokens) + if parts: + self._import(0, parts) + if token is not None and token.string == "as": + next(tokens, None) + token = next(tokens, None) + if token is None or token.string != ",": + return token + + def _from_import( + self, tokens: Iterator[tokenize.TokenInfo] + ) -> tokenize.TokenInfo | None: + level = 0 + token = next(tokens, None) + while token is not None and token.string in (".", "..."): + level += len(token.string) + token = next(tokens, None) + module: list[str] = [] + # `import` is a NAME token too, and `from . import x` has no module before it. + if token is not None and token.string != "import": + module, token = _chain(token, tokens) + # `yield from` and `raise ... from` have no `import` after them. + if token is None or token.string != "import" or not (module or level): + return token + token = next(tokens, None) + if token is not None and token.string == "(": + token = next(tokens, None) + while token is not None: + if token.string == "*": + token = next(tokens, None) + elif token.type == tokenize.NAME: + # The name may be a module of its own, or something the module defines. + self._import(level, [*module, token.string]) + token = next(tokens, None) + if token is not None and token.string == "as": + next(tokens, None) + token = next(tokens, None) + else: + break + if token is None or token.string != ",": + break + token = next(tokens, None) + self._import(level, module) + return token + + def _import(self, level: int, parts: list[str]) -> None: + key = (level, tuple(parts[:_MAX_PARTS])) + if level == 0 and any( + key[1][: len(module)] == module for module in _PROVIDER_MODULES + ): + self.provider = True + if ( + len(self.imports) < _NAMES_PER_FILE + and sum(len(part) + 1 for part in key[1]) <= _MODULE_PATH_MAX + ): + self.imports[key] = None + + +def _python_facts(data: bytes, shadowed: frozenset[str]) -> _Facts: + """Raises for a file that has to count as unread.""" + if _has_long_line(data): + raise _Unread() + work = _FileWork() + scan = _FileScan(shadowed, frozenset(), work) # type: ignore[arg-type] + if _scan_file(data, scan, read_uses=True): + raise _Unread() + own = _OwnReader() + with warnings.catch_warnings(): + # On 3.12 `tokenize` and `literal_eval` warn once per `\{`, which floods stderr. + warnings.simplefilter("ignore") + own.read(_visible_tokens(data, work)) # type: ignore[arg-type] + builds = sorted( + (framework, line) + for (framework, strength), line in scan.found.items() + if strength == "builds" + ) + models = sorted( + (framework, line) + for (framework, strength), line in scan.found.items() + if strength == "model" + ) + openai_agents = min( + ( + line + for (framework, _), line in scan.found.items() + if framework == "openai_agents" + ), + default=0, + ) + return _Facts( + builds=tuple(builds), + models=tuple(models), + found_any=bool(scan.found), + openai_agents=openai_agents, + provider=own.provider, + clients=tuple( + sorted(own.clients.items(), key=lambda item: (item[1], item[0])) + ), + env=tuple(sorted(own.env)), + tools=own.tools, + imports=tuple(own.imports), + ) + + +def _closes(text: str, quote: str) -> bool: + if quote == '"': + # `\\` is a backslash, not an escape, so a quote right after one still closes. + return _DOUBLE_QUOTE_CLOSE.search(text.replace("\\\\", "")) is not None + if quote == "`": + # Node's `.env` reader holds a backtick value open over lines, and closes it as it + # closes a double-quoted one. + return _BACKTICK_CLOSE.search(text.replace("\\\\", "")) is not None + return quote in text + + +def _looks_encoded(name: str) -> bool: + # A key wrapped over lines with no quotes and no block markers can leave a line looking + # like `=`. Setting names are words: short, or one case, or split by `_`. Encoded + # text has a long run mixing two of lower case, upper case and digits; base64url puts `_` + # inside it, and base32 has no lower case, so each run between `_` is judged alone. + for part in name.split("_"): + kinds = ( + any(char.islower() for char in part) + + any(char.isupper() for char in part) + + any(char.isdigit() for char in part) + ) + if len(part) >= _ENCODED_LENGTH and kinds >= 2: + return True + return False + + +def _continues(value: str) -> bool: + # An odd run of backslashes at the end: the last one joins the next line to the value, + # as a shell reads it. + return (len(value) - len(value.rstrip("\\"))) % 2 == 1 + + +def _template_text(data: bytes) -> str: + # `utf-8-sig`, so a byte-order mark isn't read as part of the first name. Names are ASCII, + # so a template saved in another encoding still gives them: UTF-16 says so with its + # byte-order mark, and Latin-1 reads any bytes at all. + try: + return data.decode("utf-8-sig") + except UnicodeDecodeError: + if data.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)): + return data.decode("utf-16") + return data.decode("latin-1") + + +def _template_names(data: bytes) -> set[str]: + """The setting names in a settings template. + + A line inside a value that spans lines is that value's text and never a name: a quoted + value left open, a pasted key's block, a line joined by a trailing backslash, or a key + wrapped over bare lines. A value never closed takes the rest of the file: names lost are + better than a value kept. So is a name that looks like encoded text. + """ + names: set[str] = set() + open_quote = "" + in_key = False + continued = False + # Whether the line above may wrap onto this one: a bare line, which is no setting, or a + # setting whose unquoted value is a long run of encoded text. + wraps = False + # Only `\n` ends a line: no `.env` reader splits on the other breaks `splitlines` knows, + # such as U+2028 or a form feed. + for line in _template_text(data).split("\n"): + line = line.rstrip("\r") + if continued: + continued = _continues(line) + wraps = True + continue + if open_quote or in_key: + if in_key and _KEY_END in line: + in_key = False + if open_quote and _closes(line, open_quote): + open_quote = "" + continue + if not line.strip() or line.lstrip().startswith("#"): + wraps = False + continue + # A comment may mention a key's marker; only one before it opens a block. + begin = line.split(" #", 1)[0].find(_KEY_BEGIN) + if begin >= 0 and _KEY_END not in line[begin:]: + in_key = True + match = _TEMPLATE_LINE.match(line) + if match is None: + wraps = True + continue + value = line[match.end() :].lstrip(" \t") + follows_wrap = wraps + # A pasted key's first line can carry trailing blanks or a note after it. + wraps = _ENCODED_VALUE.fullmatch(value.split(" #", 1)[0].rstrip(" \t")) is not None + quoted = value[:1] in ("'", '"', "`") + if quoted and not _closes(value[1:], value[0]): + open_quote = value[0] + continued = not quoted and _continues(value) + if follows_wrap and not value.strip(): + # `=` straight below a wrapped value is that value's last line, its padding + # read as a setting with no value. + continue + name = match.group(1) + if _good_env_name(name) and not _looks_encoded(name) and len(names) < _NAMES_PER_FILE: + names.add(name) + return names + + +def _script_names(data: bytes) -> set[str]: + names: set[str] = set() + for match in _PROCESS_ENV.finditer(data.decode("utf-8")): + name = next(group for group in match.groups() if group is not None) + if _good_env_name(name) and len(names) < _NAMES_PER_FILE: + names.add(name) + return names + + +class _Projects: + """Which folders hold a project file, looked up once per folder.""" + + def __init__(self, top: str, names: tuple[str, ...]) -> None: + self._top = top + self._names = names + self._file: dict[str, str | None] = {} + + def file_in(self, folder: str) -> str | None: + if folder not in self._file: + found = None + for name in self._names: + path = os.path.join(self._top, folder, name) + if os.path.isfile(path) and not os.path.islink(path): + found = name + break + self._file[folder] = found + return self._file[folder] + + def nearest(self, folder: str) -> str | None: + while True: + if self.file_in(folder) is not None: + return folder + if not folder: + return None + folder = posixpath.dirname(folder) + + +def _sign(path: str, facts: _Facts, graph_files: dict[str, str]) -> Sign | None: + if facts.builds: + return "framework" + if path in graph_files: + return "graph" + if posixpath.basename(path) == "agent.py" and (facts.found_any or facts.provider): + return "agent_py" + if facts.models: + return "model" + if facts.clients: + return "client" + return None + + +def _evidence( + entry: str, + facts: _Facts | None, + *, + graph: str | None, + agent_py: bool, + project: tuple[str, str] | None, + model: bool = False, +) -> tuple[tuple[str, str, int], ...]: + found: list[tuple[str, str, int]] = [] + if facts is not None: + found += [(f"builds {name}", entry, line) for name, line in facts.builds] + if graph is not None: + found.append(("graph", graph, 0)) + if agent_py: + found.append(("agent_py", entry, 0)) + if model and facts is not None: + found += [(f"model {name}", entry, line) for name, line in facts.models] + if facts is not None: + found += [(f"client {sdk}", entry, line) for sdk, line in facts.clients] + if project is not None: + found.append((f"project {project[1]}", _join(project[0], project[1]), 0)) + if facts is not None and facts.openai_agents: + found.append(("openai_agents_set_aside", entry, facts.openai_agents)) + return tuple(found) + + +def _components(nodes: list[str], edges: dict[str, list[str]]) -> list[list[str]]: + """Strongly connected components, by Tarjan's method without recursion.""" + index: dict[str, int] = {} + low: dict[str, int] = {} + stack: list[str] = [] + on_stack: set[str] = set() + components: list[list[str]] = [] + for root in nodes: + if root in index: + continue + work = [(root, 0)] + while work: + node, position = work.pop() + if position == 0: + index[node] = low[node] = len(index) + stack.append(node) + on_stack.add(node) + children = edges.get(node, []) + descended = False + for at in range(position, len(children)): + child = children[at] + if child not in index: + work.append((node, at + 1)) + work.append((child, 0)) + descended = True + break + if child in on_stack: + low[node] = min(low[node], index[child]) + if descended: + continue + if low[node] == index[node]: + component = [] + while True: + member = stack.pop() + on_stack.discard(member) + component.append(member) + if member == node: + break + components.append(component) + if work: + parent = work[-1][0] + low[parent] = min(low[parent], low[node]) + return components + + +class _Walk: + """One walk of the source. Holds every file's facts until the agents are decided.""" + + def __init__( + self, + source: Path, + *, + deadline: float, + clock: Callable[[], float], + hazards: dict[str, Hazard] | None = None, + ) -> None: + self.source = Path(source) + self.top = os.fspath(source) + self.deadline = deadline + self.clock = clock + self.hazards: dict[str, Hazard] = {} if hazards is None else hazards + self.facts: dict[str, _Facts] = {} + self.graphs: dict[str, str] = {} + self.templates: dict[str, set[str]] = {} + self.prompt_paths: list[str] = [] + self.unread = 0 + self.files_read = 0 + self.bytes_read = 0 + self.stopped: Literal["", "limit", "clock", "memory"] = "" + self.source_outside_side = False + self.script_outside_side = False + self.projects = _Projects(self.top, _PROJECT_FILES) + self._shadowed: dict[str, frozenset[str]] = {} + self._roots: dict[str, list[str]] = {} + self._edges: dict[str, tuple[str, ...]] = {} + + def run(self) -> None: + for directory, directories, files in os.walk(self.top, followlinks=False): + # Sorted, so that a walk stopped by a limit stops at the same file. + directories.sort() + files.sort() + relative = directory[len(self.top) :].split(os.sep)[1:] + if relative and any(name.endswith(".dist-info") for name in directories): + # An installed tree, not the customer's code. + directories[:] = [] + continue + directories[:] = _folders_to_enter( + directory, directories, at_top=not relative + ) + folder = "/".join(relative) + for name in files: + if self.clock() > self.deadline: + self.stopped = "clock" + return + self._take(directory, folder, name) + if self.stopped: + return + + def _take(self, directory: str, folder: str, name: str) -> None: + path = _join(folder, name) + full = os.path.join(directory, name) + if name.endswith(_SOURCE_SUFFIXES) and not _is_side_path(path): + self.source_outside_side = True + if _is_written_script(path) and not _is_side_path(path): + self.script_outside_side = True + if ( + "prompt" in path.lower() + and len(self.prompt_paths) < _MAX_PROMPT_PATHS + and not os.path.islink(full) + ): + self.prompt_paths.append(path) + if not (name.endswith(".py") or name == _LANGGRAPH or name in _TEMPLATES): + return + # Every read of one file sits in this one `try`: a file the tokenizer, the JSON + # reader or the decoder rejects is that file's problem, never the walk's. + try: + data = self._read(full) + if data is None: + return + if name.endswith(".py"): + self.facts[path] = _python_facts(data, self._shadowed_for(folder)) + elif name == _LANGGRAPH: + self._read_graphs(folder, path, data) + else: + self.templates.setdefault(folder, set()).update(_template_names(data)) + except MemoryError: + raise + except Exception: + self.unread += 1 + + def _read(self, path: str) -> bytes | None: + if self.files_read >= _MAX_FILES: + self.stopped = "limit" + return None + data = _read_file(path, _FILE_BYTES + 1) + self.files_read += 1 + self.bytes_read += len(data) + if self.bytes_read > _MAX_BYTES: + self.stopped = "limit" + return None + if len(data) > _FILE_BYTES: + raise _Unread() + return data + + def _read_graphs(self, folder: str, path: str, data: bytes) -> None: + document = json.loads(data.decode("utf-8")) + graphs = document.get("graphs") if isinstance(document, dict) else None + if not isinstance(graphs, dict): + return + for declaration in list(graphs.values())[:_NAMES_PER_FILE]: + if not isinstance(declaration, str) or ":" not in declaration: + continue + file_name = declaration.rsplit(":", 1)[0] + while file_name.startswith("./"): + file_name = file_name[2:] + target = posixpath.normpath(_join(folder, file_name)) + if target not in (".", "..") and not target.startswith(("../", "/")): + self.graphs.setdefault(target, path) + + def _shadowed_for(self, folder: str) -> frozenset[str]: + project = self.projects.nearest(folder) or "" + if project not in self._shadowed: + self._shadowed[project] = _own_modules( + Path(self.top, project) if project else Path(self.top), set() + ) + return self._shadowed[project] + + def _import_roots(self, folder: str) -> list[str]: + """Where an absolute import from ``folder`` is looked for, nearest first.""" + if folder not in self._roots: + roots: list[str] = [] + current = folder + while True: + if not current or self.projects.file_in(current) is not None: + roots += [current, _join(current, "src")] + if not current: + break + current = posixpath.dirname(current) + self._roots[folder] = roots + return self._roots[folder] + + def _module_file(self, base: str, parts: tuple[str, ...]) -> str | None: + stem = _join(base, *parts) + for path in (f"{stem}.py" if parts else "", _join(stem, "__init__.py")): + if path and path in self.facts: + return path + return None + + def _imported(self, path: str) -> tuple[str, ...]: + """The read files ``path`` imports. An import that matches none is ignored.""" + if path not in self._edges: + folder = posixpath.dirname(path) + found: dict[str, None] = {} + for level, parts in self.facts[path].imports: + target = None + if level == 0: + if parts: + for root in self._import_roots(folder): + target = self._module_file(root, parts) + if target is not None: + break + else: + base: str | None = folder + for _ in range(level - 1): + base = posixpath.dirname(base) if base else None + if base is None: + break + if base is not None: + target = self._module_file(base, parts) + if target is not None and target != path: + found[target] = None + self._edges[path] = tuple(found) + return self._edges[path] + + def _reached(self, start: str, candidates: dict[str, Sign]) -> list[str]: + """The candidates ``start`` imports, through files that aren't candidates.""" + reached: list[str] = [] + seen = {start} + frontier = [start] + for _ in range(_IMPORT_DEPTH): + following: list[str] = [] + for path in frontier: + for target in self._imported(path): + if target in seen: + continue + seen.add(target) + if target in candidates: + reached.append(target) + else: + following.append(target) + frontier = following + if not frontier: + break + return sorted(reached) + + def _absorbs(self, importer: str, imported: str, signs: dict[str, Sign]) -> bool: + return _STRENGTH[signs[importer]] >= _STRENGTH[signs[imported]] and not ( + _is_side_path(importer) and not _is_side_path(imported) + ) + + def agents(self, signs: dict[str, Sign]) -> list[_Agent]: + nodes = sorted(signs) + edges = { + node: [ + target + for target in self._reached(node, signs) + if self._absorbs(node, target, signs) + ] + for node in nodes + } + component_of: dict[str, int] = {} + components = _components(nodes, edges) + for number, members in enumerate(components): + for member in members: + component_of[member] = number + taken = { + component_of[target] + for node in nodes + for target in edges[node] + if component_of[target] != component_of[node] + } + entries = [ + min(members, key=lambda path: (-_STRENGTH[signs[path]], len(path), path)) + for number, members in enumerate(components) + if number not in taken + ] + stronger = [ + (self._project_of(path), _is_side_path(path)) + for path, sign in signs.items() + if sign not in _WEAKEST + ] + kept = [] + for entry in sorted(entries): + side = _is_side_path(entry) + if signs[entry] in _WEAKEST: + project = self._project_of(entry) + # A side candidate never decides for a file outside side folders. + if any( + other == project and (side or not other_side) + for other, other_side in stronger + ): + continue + kept.append((entry, side)) + return [ + _Agent(entry, signs[entry], side, self._subfolder(entry, side, kept)) + for entry, side in kept + ] + + def _project_of(self, path: str) -> str: + return self.projects.nearest(posixpath.dirname(path)) or "" + + def _subfolder(self, entry: str, side: bool, agents: list[tuple[str, bool]]) -> str: + folder = posixpath.dirname(entry) + if side: + return folder + project = self.projects.nearest(folder) + if project is None: + return folder + sharing = sum( + 1 + for other, other_side in agents + if not other_side and _under(other, project) + ) + return project if sharing == 1 else folder + + def candidate( + self, agent: _Agent, listed: list[_Agent], *, upload: bool + ) -> Candidate: + subfolder = "" if upload else agent.subfolder + deeper: list[str] = [] + if not upload: + # An upload lists one agent, so nothing else claims a folder under the root. + deeper = [ + other.subfolder + for other in listed + if other.subfolder != subfolder and _under(other.subfolder, subfolder) + ] + own = [agent.entry] if agent.entry in self.facts else [] + for path in self.facts: + if ( + path != agent.entry + and _under(path, subfolder) + and not any(_under(path, folder) for folder in deeper) + and not _is_side_path(_below(path, subfolder)) + ): + own.append(path) + code = sorted({name for path in own for name in self.facts[path].env}) + templates = self.templates.get(subfolder, set()) | self.templates.get("", set()) + facts = self.facts.get(agent.entry) + project_file = self.projects.file_in(subfolder) + return Candidate( + entry=agent.entry, + subfolder=subfolder, + sign=agent.sign, + # The subfolder is always the entry's folder or one above it, so the + # entry's path holds every part of it. + side=agent.side, + builds=tuple(name for name, _ in facts.builds) if facts else (), + build_may_stop=build_hazard( + self.source, + subfolder, + deadline=self.deadline, + clock=self.clock, + hazards=self.hazards, + ), + partial=False, + env_names=(*code, *sorted(templates - set(code))), + tools=sum(self.facts[path].tools for path in own), + prompt_files=_prompt_count(self.prompt_paths, subfolder), + evidence=_evidence( + agent.entry, + facts, + graph=self.graphs.get(agent.entry), + agent_py=agent.sign == "agent_py", + model=agent.sign == "model", + project=(subfolder, project_file) if project_file else None, + ), + ) + + def result(self, *, upload: bool) -> WalkResult: + signs: dict[str, Sign] = {} + for path, facts in self.facts.items(): + sign = _sign(path, facts, self.graphs) + if sign is not None: + signs[path] = sign + left_out = 0 + agents = [] + for agent in self.agents(signs): + if fits_path_rule(agent.entry) and ( + not agent.subfolder or fits_path_rule(agent.subfolder) + ): + agents.append(agent) + else: + left_out += 1 + agents.sort( + key=lambda a: (a.side, -_STRENGTH[a.sign], a.subfolder, a.entry) + ) + listed = agents[:1] if upload else agents[:_MAX_AGENTS] + outside = {sign for path, sign in signs.items() if not _is_side_path(path)} + return WalkResult( + agents=tuple( + self.candidate(agent, listed, upload=upload) for agent in listed + ), + more_than_fifty=not upload and len(agents) > _MAX_AGENTS, + stopped=self.stopped, + unread_files=self.unread, + source_outside_side=self.source_outside_side, + candidate_outside_side=any(not _is_side_path(path) for path in signs), + script_outside_side=self.script_outside_side, + only_weak_outside_side=bool(outside) and outside <= _WEAKEST, + paths_left_out=left_out, + upload_others=len(agents) - len(listed) if upload else 0, + upload_others_side=sum(agent.side for agent in agents[len(listed) :]) + if upload + else 0, + ) + + +def _prompt_count(paths: list[str], subfolder: str) -> int: + return sum( + 1 + for path in paths + if _under(path, subfolder) and "prompt" in _below(path, subfolder).lower() + ) + + +def _empty_result(walk: _Walk, stopped: Literal["limit", "memory"]) -> WalkResult: + return WalkResult( + agents=(), + more_than_fifty=False, + stopped=stopped, + unread_files=walk.unread, + source_outside_side=walk.source_outside_side, + candidate_outside_side=False, + script_outside_side=False, + only_weak_outside_side=False, + paths_left_out=0, + upload_others=0, + upload_others_side=0, + ) + + +def walk_repo( + source: Path, + *, + deadline: float, + upload: bool = False, + clock: Callable[[], float] = time.monotonic, + hazards: dict[str, Hazard] | None = None, +) -> WalkResult: + """``hazards`` is one scan's answers from ``build_hazard``, shared with ``describe_entry``.""" + walk = _Walk(source, deadline=deadline, clock=clock, hazards=hazards) + try: + walk.run() + except MemoryError: + # Only finished rows survive a memory error, and the walk has none yet. + walk.facts.clear() + walk.graphs.clear() + walk.templates.clear() + walk.prompt_paths.clear() + logger.warning("agent walk: stopped=memory unread=%d", walk.unread) + return _empty_result(walk, "memory") + except Exception as exc: + # A fault outside any one file; what was read before it stands. + logger.warning("agent walk: stopped=limit class=%s", type(exc).__name__) + walk.stopped = "limit" + try: + result = walk.result(upload=upload) + except MemoryError: + raise + except Exception as exc: + logger.warning("agent walk failed: class=%s", type(exc).__name__) + result = _empty_result(walk, "limit") + logger.info( + "agent walk: files_read=%d unread=%d agents=%d left_out=%d stopped=%s", + walk.files_read, + result.unread_files, + len(result.agents), + result.paths_left_out, + result.stopped or "no", + ) + return result + + +def build_hazard( + source: Path, + subfolder: str, + *, + deadline: float = math.inf, + clock: Callable[[], float] = time.monotonic, + hazards: dict[str, Hazard] | None = None, +) -> Hazard: + """Why a build of the folder may stop on its own fallback, or ``""``. + + ``hazards`` holds one scan's answers by folder, so a folder many agents share is looked + through once. The look stops, with ``""``, at the deadline or after ``_HAZARD_ENTRIES`` + entries: it walks folders the scan skips, which the walk's own clock never saw. + """ + if hazards is not None and subfolder in hazards: + return hazards[subfolder] + hazard = _hazard(source, subfolder, deadline=deadline, clock=clock) + if hazards is not None: + hazards[subfolder] = hazard + return hazard + + +def _hazard( + source: Path, subfolder: str, *, deadline: float, clock: Callable[[], float] +) -> Hazard: + top = os.fspath(source) + if _has_link(top, subfolder): + return "" + folder = os.path.join(top, subfolder) if subfolder else top + try: + data = _read_file(os.path.join(folder, _LANGGRAPH), _FILE_BYTES + 1) + document = json.loads(data.decode("utf-8")) + graphs = document.get("graphs") if isinstance(document, dict) else None + if isinstance(graphs, dict) and len(graphs) > 1: + return "graphs" + except MemoryError: + raise + except Exception: + pass + # The build counts every `agent.py` below, side folders and the scan's skipped + # folders included, ignoring only its own short list. + found = 0 + entries = 0 + try: + for index, (_, directories, files) in enumerate( + os.walk(folder, followlinks=False) + ): + # The folder's own listing is one read and always looked at; the clock bounds + # the folders below it. + if index and clock() > deadline: + return "" + directories[:] = [ + name for name in directories if name not in _BUILD_IGNORED + ] + found += files.count("agent.py") + if found > 1: + return "agent_py" + entries += len(directories) + len(files) + if entries > _HAZARD_ENTRIES: + return "" + except MemoryError: + raise + except Exception: + return "" + return "" + + +class _Description: + """The reads behind one AI-found agent, bounded by its own caps and the clock.""" + + def __init__( + self, + top: str, + entry: str, + subfolder: str, + *, + deadline: float, + clock: Callable[[], float], + ) -> None: + self.top = top + self.entry = entry + self.subfolder = subfolder + self.deadline = deadline + self.clock = clock + self.projects = _Projects(top, _PROJECT_FILES) + self.facts: dict[str, _Facts] = {} + self.script_env: set[str] = set() + self.templates: set[str] = set() + self.prompt_files = 0 + self.files_read = 0 + self.bytes_read = 0 + self.partial = False + self._shadowed: dict[str, frozenset[str]] = {} + + def run(self) -> None: + if _has_link(self.top, self.subfolder): + # Nothing under it is read, so its counts are short. + self.partial = True + return + for folder in dict.fromkeys((self.subfolder, "")): + for name in _TEMPLATES: + if self._out_of_time(): + return + data = self._read(os.path.join(self.top, folder, name)) + if self.partial: + return + if data is not None: + try: + self.templates |= _template_names(data) + except MemoryError: + raise + except Exception: + pass + start = os.path.join(self.top, self.subfolder) if self.subfolder else self.top + for directory, directories, files in os.walk(start, followlinks=False): + directories.sort() + files.sort() + relative = directory[len(self.top) :].split(os.sep)[1:] + if relative and any(name.endswith(".dist-info") for name in directories): + directories[:] = [] + continue + directories[:] = _folders_to_enter( + directory, directories, at_top=not relative + ) + folder = "/".join(relative) + for name in files: + if self._out_of_time(): + return + self._take(directory, folder, name) + if self.partial: + return + + def _out_of_time(self) -> bool: + if self.clock() > self.deadline: + self.partial = True + return self.partial + + def _take(self, directory: str, folder: str, name: str) -> None: + path = _join(folder, name) + full = os.path.join(directory, name) + below = _below(path, self.subfolder) + if "prompt" in below.lower() and not os.path.islink(full): + self.prompt_files += 1 + if path != self.entry and _is_side_path(below): + return + if not name.endswith((".py", *_SCRIPT_SUFFIXES)): + return + data = self._read(full) + if data is None: + return + try: + if name.endswith(".py"): + self.facts[path] = _python_facts(data, self._shadowed_for(folder)) + else: + self.script_env |= _script_names(data) + except MemoryError: + raise + except Exception: + pass + + def _read(self, path: str) -> bytes | None: + """The file's bytes, or None for a file that can't or mustn't be read; sets + ``partial`` when a cap is reached.""" + if self.files_read >= _DESCRIBE_FILES: + self.partial = True + return None + try: + data = _read_file(path, _FILE_BYTES + 1) + except MemoryError: + raise + except Exception: + return None + self.files_read += 1 + self.bytes_read += len(data) + if self.bytes_read > _DESCRIBE_BYTES: + self.partial = True + return None + return data if len(data) <= _FILE_BYTES else None + + def _shadowed_for(self, folder: str) -> frozenset[str]: + project = self.projects.nearest(folder) or "" + if project not in self._shadowed: + self._shadowed[project] = _own_modules( + Path(self.top, project) if project else Path(self.top), set() + ) + return self._shadowed[project] + + +def describe_entry( + source: Path, + entry: str, + folder: str, + *, + deadline: float, + upload: bool = False, + clock: Callable[[], float] = time.monotonic, + hazards: dict[str, Hazard] | None = None, +) -> Candidate | None: + """The row for one file the AI step kept; None only when its path is refused.""" + if not fits_path_rule(entry): + return None + top = os.fspath(source) + entry_folder = posixpath.dirname(entry) + projects = _Projects(top, (*_PROJECT_FILES, _PACKAGE_FILE)) + if upload: + subfolder = "" + elif (not folder or fits_path_rule(folder)) and _under(entry, folder): + subfolder = folder + else: + nearest = projects.nearest(entry_folder) + subfolder = entry_folder if nearest is None else nearest + reading = _Description(top, entry, subfolder, deadline=deadline, clock=clock) + try: + reading.run() + except MemoryError: + raise + except Exception as exc: + # A fault outside any one file: what was read stands, and the row says so. + logger.warning("agent description stopped: class=%s", type(exc).__name__) + reading.partial = True + facts = reading.facts.get(entry) + code = {name for found in reading.facts.values() for name in found.env} + code |= reading.script_env + project_file = projects.file_in(subfolder) + if not reading.partial: + logger.info("agent description: files_read=%d", reading.files_read) + else: + logger.info("agent description: files_read=%d partial=yes", reading.files_read) + return Candidate( + entry=entry, + subfolder=subfolder, + sign="client", + side=_is_side_path(entry), + builds=tuple(name for name, _ in facts.builds) if facts else (), + build_may_stop=build_hazard( + source, subfolder, deadline=deadline, clock=clock, hazards=hazards + ), + partial=reading.partial, + env_names=(*sorted(code), *sorted(reading.templates - code)), + tools=sum(found.tools for found in reading.facts.values()), + prompt_files=reading.prompt_files, + evidence=_evidence( + entry, + facts, + graph=None, + agent_py=False, + project=(subfolder, project_file) if project_file else None, + ), + ) diff --git a/src/fi/alk/harness/skills/find-agents/SKILL.md b/src/fi/alk/harness/skills/find-agents/SKILL.md new file mode 100644 index 000000000..2f3bbd763 --- /dev/null +++ b/src/fi/alk/harness/skills/find-agents/SKILL.md @@ -0,0 +1,86 @@ +--- +name: find-agents +description: Reads a repository in which a plain code check found no agent outside test and example folders, apart from any files that only call a model, and names the folder and entry file of each runnable AI agent it holds, including agents in TypeScript or JavaScript, which that check can't see. +--- + +# Find the agents in a repository + +A plain code check has already looked at this repository and found no agent outside test and +example folders, apart from any files that only call a model, which are already listed. It can't +see agents in TypeScript or JavaScript. You are the second look. Your answer is a list of agents, each a folder and the +entry file that runs it. Code checks every pair you give: a path that isn't a file in the +repository is refused, so a guess costs you a turn and adds nothing. + +Nobody is reading along and nobody will answer a question. Work, submit, end your turn. + +## Your tools + +You have exactly three tools for reading, and one for answering: + +- `mcp__scan_files__read`: one file, by `path`, with optional `offset` (first line, from 1) and + `limit` (at most 400 lines). +- `mcp__scan_files__glob`: file names matching `pattern` (for example `*.py` or `**/agent*.ts`), + under an optional `path`. +- `mcp__scan_files__grep`: lines containing `text`, under an optional `path`, with an optional + `glob` for file names and `ignore_case`. `text` is plain text, matched exactly as written. It + is not a regular expression: `OpenAI(` finds `OpenAI(`, and `a|b` finds only `a|b`. +- `mcp__scan__submit_agents`: your answer. + +Any other tool the host shows you (`Read`, `Grep`, `Bash` and the rest) is refused and wastes a +turn. Paths are relative to the repository's root. Some files are hidden on purpose: settings, +keys and credentials. Don't try to reach them. + +A result that ends "Stopped early at the search limit" is incomplete: search a narrower `path` +before you conclude something isn't there. + +## What an agent looks like + +An agent is code that runs a model in a loop or a graph: it sends a model messages, gives it +tools or steps, and acts on what comes back. Its entry is the file you would run, or the file +that builds the agent object something else serves. + +- **Python:** a file that builds an agent with a framework (LangGraph, LangChain, CrewAI, + LlamaIndex, Pydantic AI, the OpenAI Agents SDK, Google ADK, AutoGen, smolagents, LiveKit and + the like), or calls a provider's client (`OpenAI(`, `Anthropic(`, `genai.Client(`, + `litellm.completion(`) in a loop with tools. Often `agent.py`, `main.py`, `app.py`, + `graph.py` or `server.py`, near a `pyproject.toml` or `requirements.txt`. +- **TypeScript and JavaScript:** a file that imports `@langchain/*`, `@openai/agents`, `ai` (the + Vercel AI SDK), `@mastra/*`, `openai` or `@anthropic-ai/sdk` and runs the loop, often + `index.ts`, `agent.ts`, `server.ts` or `src/main.ts`, near a `package.json`. +- **Notebooks:** an `.ipynb` whose cells build and run an agent counts only if nothing else in + its folder does; prefer a `.py` or `.ts` file beside it. + +The folder is the agent's own folder: the one holding its project file, or else the entry's own +folder. Use `""` for the repository's root. + +## What to skip + +Skip tests, examples, samples, demos, docs, notebooks of experiments, benchmarks, vendored and +third-party code, built or generated output (`dist`, `build`, `*.min.js`, generated clients), +and libraries the repository only installs. A helper that calls a model once, with no loop and +no tools, is not an agent. Several entry files that run the same agent are one agent: name the +one you would run. + +## How to work + +Start broad and narrow down. `glob` for project files and likely entry names, `grep` for the +imports and client calls above, then `read` the few files that decide it. Don't read files whose +answer you already have. + +## Answering + +Call `mcp__scan__submit_agents` once, with every agent you found: + +```json +{"agents": [{"folder": "services/support", "entry": "services/support/agent.py"}]} +``` + +- `entry` is a file inside `folder`, both relative to the root. At most 50. +- If there are none, submit an empty list: `{"agents": []}`. +- If the reply says pairs were refused and none kept, fix the paths and submit again. +- Once a submit is recorded, the work is done: end your turn. Further calls change nothing. + +A turn that ends without a submit is the end of the step, and nothing you found is kept. + +Never quote file content in anything you write: not code, not settings, not values. Name files by +their path and nothing more. diff --git a/tests/harness/fixtures/agent_scan_e07d56fcd.json b/tests/harness/fixtures/agent_scan_e07d56fcd.json new file mode 100644 index 000000000..8be1142ed --- /dev/null +++ b/tests/harness/fixtures/agent_scan_e07d56fcd.json @@ -0,0 +1,625 @@ +{ + "serializer_commit": "e07d56fcd", + "cases": [ + { + "name": "found_one_agent", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "none_no_agents", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"none\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "error_with_error", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\", \"message\": \"The scan ran out of time.\"}}", + "accepted": true + }, + { + "name": "top_level_not_an_object", + "document_text": "[{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}]", + "accepted": false + }, + { + "name": "schema_version_other", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v2\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "schema_version_missing", + "document_text": "{\"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "status_unknown_word", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"partial\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "status_trailing_space", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found \", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "status_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "found_without_agents", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "found_with_error", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\", \"message\": \"The scan ran out of time.\"}}", + "accepted": false + }, + { + "name": "none_with_agents", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"none\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "none_with_error", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"none\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\", \"message\": \"The scan ran out of time.\"}}", + "accepted": false + }, + { + "name": "error_without_error", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "error_with_agents", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\", \"message\": \"The scan ran out of time.\"}}", + "accepted": false + }, + { + "name": "commit_null", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": null, \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "commit_uppercase", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789ABCDEF0123456789ABCDEF01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "commit_39", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef0123456\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "commit_41", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef012345670\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "commit_empty", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "commit_not_hex", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"gggggggggggggggggggggggggggggggggggggggg\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "commit_trailing_line_break", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\\n\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "commit_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "agents_50", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a0.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a1.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a2.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a3.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a4.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a5.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a6.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a7.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a8.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a9.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a10.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a11.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a12.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a13.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a14.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a15.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a16.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a17.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a18.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a19.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a20.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a21.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a22.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a23.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a24.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a25.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a26.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a27.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a28.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a29.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a30.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a31.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a32.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a33.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a34.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a35.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a36.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a37.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a38.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a39.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a40.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a41.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a42.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a43.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a44.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a45.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a46.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a47.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a48.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a49.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "agents_51", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a0.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a1.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a2.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a3.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a4.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a5.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a6.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a7.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a8.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a9.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a10.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a11.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a12.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a13.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a14.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a15.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a16.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a17.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a18.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a19.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a20.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a21.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a22.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a23.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a24.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a25.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a26.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a27.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a28.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a29.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a30.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a31.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a32.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a33.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a34.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a35.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a36.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a37.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a38.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a39.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a40.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a41.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a42.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a43.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a44.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a45.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a46.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a47.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a48.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a49.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a50.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "agents_not_a_list", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": {\"0\": {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}}, \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "agents_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "agent_not_an_object", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [\"support\"], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "truncated_true", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": true, \"error\": null}", + "accepted": true + }, + { + "name": "truncated_null", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": null, \"error\": null}", + "accepted": false + }, + { + "name": "truncated_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"error\": null}", + "accepted": false + }, + { + "name": "error_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false}", + "accepted": false + }, + { + "name": "error_code_64", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\", \"message\": \"The scan ran out of time.\"}}", + "accepted": true + }, + { + "name": "error_code_65", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\", \"message\": \"The scan ran out of time.\"}}", + "accepted": false + }, + { + "name": "error_code_uppercase", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"Scan\", \"message\": \"The scan ran out of time.\"}}", + "accepted": false + }, + { + "name": "error_code_hyphen", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"scan-x\", \"message\": \"The scan ran out of time.\"}}", + "accepted": false + }, + { + "name": "error_code_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"message\": \"The scan ran out of time.\"}}", + "accepted": false + }, + { + "name": "error_message_empty", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\", \"message\": \"\"}}", + "accepted": false + }, + { + "name": "error_message_blank", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\", \"message\": \" \"}}", + "accepted": false + }, + { + "name": "error_message_2000", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\", \"message\": \"mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm\"}}", + "accepted": true + }, + { + "name": "error_message_2001", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\", \"message\": \"mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm\"}}", + "accepted": false + }, + { + "name": "error_message_null_byte", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\", \"message\": \"a\\u0000b\"}}", + "accepted": false + }, + { + "name": "error_message_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": {\"code\": \"scan_timed_out\"}}", + "accepted": false + }, + { + "name": "error_not_an_object", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"error\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [], \"truncated\": false, \"error\": \"scan_timed_out\"}", + "accepted": false + }, + { + "name": "name_255", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "name_256", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "name_255_astral", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "name_256_astral", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\\ud83d\\ude00\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "name_255_and_a_trailing_space", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"nnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnnn \", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "name_empty", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "name_ideographic_spaces_only", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"\\u3000\\u2003\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "name_inner_unicode_spaces", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"a\\u3000b\\u2003c\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "name_null_byte", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"sup\\u0000port\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "name_lone_surrogate", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"sup\\ud800port\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "name_trailing_line_break", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\\n\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "name_null", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": null, \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "name_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_root", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "subfolder_blank", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"\\u3000\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "subfolder_512", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/bc\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "subfolder_513", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/bcd\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_leading_slash", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"/services\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_dot_dot", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/../x\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_dot", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/./x\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_only_dot_dot", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"..\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_three_dots", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"...\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "subfolder_double_slash", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services//x\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_trailing_slash", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_space", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"my services\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_non_ascii", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"servi\\u00e7es\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_trailing_line_break", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services\\n\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "subfolder_null", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": null, \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "subfolder_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "entry_512", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/bc\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "entry_513", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/bcd\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "entry_empty", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "entry_backslash", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services\\\\graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "entry_null_byte", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"graph\\u0000.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "entry_lone_surrogate", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"graph\\udcff.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "entry_trailing_line_break", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\\n\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "entry_inner_line_break", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services\\ngraph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "entry_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_label_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_version_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_null", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": null, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_unknown_word", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"autogen\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_each_word", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"langchain.py\", \"framework\": {\"name\": \"langchain\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"langgraph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"livekit.py\", \"framework\": {\"name\": \"livekit\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"claude_agent_sdk.py\", \"framework\": {\"name\": \"claude_agent_sdk\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"openai_agents.py\", \"framework\": {\"name\": \"openai_agents\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"pydantic_ai.py\", \"framework\": {\"name\": \"pydantic_ai\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"google_adk.py\", \"framework\": {\"name\": \"google_adk\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"crewai.py\", \"framework\": {\"name\": \"crewai\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"pipecat.py\", \"framework\": {\"name\": \"pipecat\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"deepagents.py\", \"framework\": {\"name\": \"deepagents\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"other.py\", \"framework\": {\"name\": \"other\", \"version\": null, \"label\": \"x\"}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"none.py\", \"framework\": {\"name\": \"none\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}, {\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"unknown.py\", \"framework\": {\"name\": \"unknown\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "framework_version_64", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": \"1111111111111111111111111111111111111111111111111111111111111111\", \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "framework_version_65", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": \"11111111111111111111111111111111111111111111111111111111111111111\", \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_version_empty", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": \"\", \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_other_with_label", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"other\", \"version\": null, \"label\": \"llama_index\"}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "framework_other_without_label", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"other\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_other_empty_label", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"other\", \"version\": null, \"label\": \"\"}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_named_with_label", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": \"llama_index\"}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "framework_label_128", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"other\", \"version\": null, \"label\": \"llllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllll\"}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "framework_label_129", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"other\", \"version\": null, \"label\": \"lllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllll\"}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "confidence_low", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"low\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "confidence_medium", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"medium\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "confidence_uppercase", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"HIGH\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "confidence_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "evidence_empty", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "evidence_5", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"e\", \"e\", \"e\", \"e\", \"e\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "evidence_6", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"e\", \"e\", \"e\", \"e\", \"e\", \"e\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "evidence_200", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "evidence_201", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "evidence_blank", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "evidence_null_byte", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"a\\u0000b\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "evidence_lone_surrogate", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"a\\udcffb\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "evidence_null_item", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [null], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "evidence_not_a_list", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": \"Builds with LangGraph.\", \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "env_names_200", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"N0\", \"N1\", \"N2\", \"N3\", \"N4\", \"N5\", \"N6\", \"N7\", \"N8\", \"N9\", \"N10\", \"N11\", \"N12\", \"N13\", \"N14\", \"N15\", \"N16\", \"N17\", \"N18\", \"N19\", \"N20\", \"N21\", \"N22\", \"N23\", \"N24\", \"N25\", \"N26\", \"N27\", \"N28\", \"N29\", \"N30\", \"N31\", \"N32\", \"N33\", \"N34\", \"N35\", \"N36\", \"N37\", \"N38\", \"N39\", \"N40\", \"N41\", \"N42\", \"N43\", \"N44\", \"N45\", \"N46\", \"N47\", \"N48\", \"N49\", \"N50\", \"N51\", \"N52\", \"N53\", \"N54\", \"N55\", \"N56\", \"N57\", \"N58\", \"N59\", \"N60\", \"N61\", \"N62\", \"N63\", \"N64\", \"N65\", \"N66\", \"N67\", \"N68\", \"N69\", \"N70\", \"N71\", \"N72\", \"N73\", \"N74\", \"N75\", \"N76\", \"N77\", \"N78\", \"N79\", \"N80\", \"N81\", \"N82\", \"N83\", \"N84\", \"N85\", \"N86\", \"N87\", \"N88\", \"N89\", \"N90\", \"N91\", \"N92\", \"N93\", \"N94\", \"N95\", \"N96\", \"N97\", \"N98\", \"N99\", \"N100\", \"N101\", \"N102\", \"N103\", \"N104\", \"N105\", \"N106\", \"N107\", \"N108\", \"N109\", \"N110\", \"N111\", \"N112\", \"N113\", \"N114\", \"N115\", \"N116\", \"N117\", \"N118\", \"N119\", \"N120\", \"N121\", \"N122\", \"N123\", \"N124\", \"N125\", \"N126\", \"N127\", \"N128\", \"N129\", \"N130\", \"N131\", \"N132\", \"N133\", \"N134\", \"N135\", \"N136\", \"N137\", \"N138\", \"N139\", \"N140\", \"N141\", \"N142\", \"N143\", \"N144\", \"N145\", \"N146\", \"N147\", \"N148\", \"N149\", \"N150\", \"N151\", \"N152\", \"N153\", \"N154\", \"N155\", \"N156\", \"N157\", \"N158\", \"N159\", \"N160\", \"N161\", \"N162\", \"N163\", \"N164\", \"N165\", \"N166\", \"N167\", \"N168\", \"N169\", \"N170\", \"N171\", \"N172\", \"N173\", \"N174\", \"N175\", \"N176\", \"N177\", \"N178\", \"N179\", \"N180\", \"N181\", \"N182\", \"N183\", \"N184\", \"N185\", \"N186\", \"N187\", \"N188\", \"N189\", \"N190\", \"N191\", \"N192\", \"N193\", \"N194\", \"N195\", \"N196\", \"N197\", \"N198\", \"N199\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "env_names_201", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"N0\", \"N1\", \"N2\", \"N3\", \"N4\", \"N5\", \"N6\", \"N7\", \"N8\", \"N9\", \"N10\", \"N11\", \"N12\", \"N13\", \"N14\", \"N15\", \"N16\", \"N17\", \"N18\", \"N19\", \"N20\", \"N21\", \"N22\", \"N23\", \"N24\", \"N25\", \"N26\", \"N27\", \"N28\", \"N29\", \"N30\", \"N31\", \"N32\", \"N33\", \"N34\", \"N35\", \"N36\", \"N37\", \"N38\", \"N39\", \"N40\", \"N41\", \"N42\", \"N43\", \"N44\", \"N45\", \"N46\", \"N47\", \"N48\", \"N49\", \"N50\", \"N51\", \"N52\", \"N53\", \"N54\", \"N55\", \"N56\", \"N57\", \"N58\", \"N59\", \"N60\", \"N61\", \"N62\", \"N63\", \"N64\", \"N65\", \"N66\", \"N67\", \"N68\", \"N69\", \"N70\", \"N71\", \"N72\", \"N73\", \"N74\", \"N75\", \"N76\", \"N77\", \"N78\", \"N79\", \"N80\", \"N81\", \"N82\", \"N83\", \"N84\", \"N85\", \"N86\", \"N87\", \"N88\", \"N89\", \"N90\", \"N91\", \"N92\", \"N93\", \"N94\", \"N95\", \"N96\", \"N97\", \"N98\", \"N99\", \"N100\", \"N101\", \"N102\", \"N103\", \"N104\", \"N105\", \"N106\", \"N107\", \"N108\", \"N109\", \"N110\", \"N111\", \"N112\", \"N113\", \"N114\", \"N115\", \"N116\", \"N117\", \"N118\", \"N119\", \"N120\", \"N121\", \"N122\", \"N123\", \"N124\", \"N125\", \"N126\", \"N127\", \"N128\", \"N129\", \"N130\", \"N131\", \"N132\", \"N133\", \"N134\", \"N135\", \"N136\", \"N137\", \"N138\", \"N139\", \"N140\", \"N141\", \"N142\", \"N143\", \"N144\", \"N145\", \"N146\", \"N147\", \"N148\", \"N149\", \"N150\", \"N151\", \"N152\", \"N153\", \"N154\", \"N155\", \"N156\", \"N157\", \"N158\", \"N159\", \"N160\", \"N161\", \"N162\", \"N163\", \"N164\", \"N165\", \"N166\", \"N167\", \"N168\", \"N169\", \"N170\", \"N171\", \"N172\", \"N173\", \"N174\", \"N175\", \"N176\", \"N177\", \"N178\", \"N179\", \"N180\", \"N181\", \"N182\", \"N183\", \"N184\", \"N185\", \"N186\", \"N187\", \"N188\", \"N189\", \"N190\", \"N191\", \"N192\", \"N193\", \"N194\", \"N195\", \"N196\", \"N197\", \"N198\", \"N199\", \"N200\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "env_name_128", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "env_name_129", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "env_name_leading_digit", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"1KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "env_name_hyphen", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI-KEY\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "env_name_trailing_line_break", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\\n\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "env_name_lone_surrogate", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"KEY\\ud800\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "env_names_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"estimates\": {\"tools\": 3, \"prompt_files\": 1}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "estimates_zero", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 0, \"prompt_files\": 0}}], \"truncated\": false, \"error\": null}", + "accepted": true + }, + { + "name": "estimates_negative", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": -1, \"prompt_files\": 0}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "estimates_bool", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": true, \"prompt_files\": 0}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "estimates_prompt_files_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"], \"estimates\": {\"tools\": 0}}], \"truncated\": false, \"error\": null}", + "accepted": false + }, + { + "name": "estimates_missing", + "document_text": "{\"schema_version\": \"futureagi.agent-scan.v1\", \"status\": \"found\", \"commit_sha\": \"0123456789abcdef0123456789abcdef01234567\", \"agents\": [{\"name\": \"support\", \"subfolder\": \"services/support\", \"entry\": \"services/support/graph.py\", \"framework\": {\"name\": \"langgraph\", \"version\": null, \"label\": null}, \"confidence\": \"high\", \"evidence\": [\"Builds with LangGraph in services/support/graph.py, line 12.\"], \"env_names\": [\"OPENAI_API_KEY\"]}], \"truncated\": false, \"error\": null}", + "accepted": false + } + ] +} diff --git a/tests/harness/test_agent_scan.py b/tests/harness/test_agent_scan.py new file mode 100644 index 000000000..cd4bfaa7d --- /dev/null +++ b/tests/harness/test_agent_scan.py @@ -0,0 +1,1175 @@ +"""The find-agents mode: what it refuses, the file it leaves on every path, and the cost it leaves.""" + +from __future__ import annotations + +import asyncio +import inspect +import json +import logging +import math +import os +import shutil +import time +from pathlib import Path + +import pytest + +from fi.alk.harness import agent_scan, reading_entry, scan_walk, spend +from fi.alk.harness.agent_scan import FIND_AGENTS, _check_job, _hints, _work +from fi.alk.harness.job import ( + FailureDomain, + HarnessJob, + HarnessStage, + RepositorySource, + SourceKind, +) +from fi.alk.harness.scan_ai import AiOutcome +from fi.alk.harness.scan_document import ScanResultInvalid, wire_problems +from fi.alk.harness.scan_folders import ClockRanOut, FolderCheckerBroken, FolderWord +from fi.alk.harness.scan_walk import Candidate, WalkResult + +GRAPH = "from langgraph.graph import StateGraph\nStateGraph(dict)\n" +SCRIPT_AGENT = ( + "import { createReactAgent } from '@langchain/langgraph/prebuilt';\n" + "export const agent = createReactAgent({ llm, tools: [] });\n" +) +MODEL_HELPER = ( + "from langchain_openai import ChatOpenAI\n" + "def judge(x):\n" + " return ChatOpenAI().invoke(x)\n" +) +# A TypeScript agent the walk can't list, beside a Python file that only calls a model. +SCRIPT_BESIDE_A_HELPER = {"web/src/agent.ts": SCRIPT_AGENT, "backend/judge.py": MODEL_HELPER} +CLIENT = "from openai import OpenAI\nOpenAI()\n" +# Both kinds of file that only call a model: a chat model, and a provider's own client. +HELPERS = {"chat_model": MODEL_HELPER, "client": CLIENT} +PLAIN = "def main():\n return 1\n" +COMMIT = "AB" * 20 +# Plain code outside the side folders and one client inside them: the walk lists only a side +# agent, so the model is asked. +SIDE_ONLY = {"app/main.py": PLAIN, "tests/test_bot.py": CLIENT} +AI_SENTENCE = "Found by the AI step, not by the code check." + + +class FakeChecker: + """Answers every folder at once. From call ``fail_at`` on (counted from 1) it raises ``error``.""" + + def __init__(self) -> None: + self.name = "langgraph" + self.fail_at: int | None = None + self.error: Exception = ClockRanOut("the clock ran out") + self.calls: list[str] = [] + + def __enter__(self) -> FakeChecker: + return self + + def __exit__(self, *exc: object) -> None: + return None + + def check(self, source, subfolder, *, deadline, clock=None) -> FolderWord: + self.calls.append(subfolder) + if self.fail_at is not None and len(self.calls) >= self.fail_at: + raise self.error + return FolderWord({"name": self.name, "version": None, "label": None}, finished=True) + + +class FakeAi: + """The model step: one turn ends, then ``during`` runs, then ``outcome`` comes back.""" + + def __init__(self) -> None: + self.outcome = _outcome("empty") + self.during = None + self.calls: list[dict] = [] + + async def __call__(self, source, *, hints, seconds, on_turn_end) -> AiOutcome: + self.calls.append({"source": source, "hints": hints, "seconds": seconds}) + on_turn_end() + if self.during is not None: + self.during() + return self.outcome + + +def _outcome( + kind, pairs=(), *, refused=0, complete=True, retryable=False, uncarried=0 +) -> AiOutcome: + return AiOutcome(kind, tuple(pairs), refused, complete, retryable, "fixed words", uncarried) + + +@pytest.fixture(autouse=True) +def _fresh_state(monkeypatch, tmp_path): + monkeypatch.delenv(spend.JOURNAL_ALIAS, raising=False) + monkeypatch.setattr(spend, "_path", None) + # The machine's own secrets files must not decide what a test sees masked. + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (tmp_path / "no-secrets.json",)) + monkeypatch.setattr(reading_entry, "_secret_values", None) + monkeypatch.setattr(agent_scan, "CLOCK", 1200.0) + monkeypatch.setattr(agent_scan, "AI_SECONDS", 600.0) + spend._stages.clear() + yield + spend._stages.clear() + + +@pytest.fixture(autouse=True) +def checker(monkeypatch) -> FakeChecker: + fake = FakeChecker() + monkeypatch.setattr(agent_scan, "FolderChecker", lambda: fake) + return fake + + +@pytest.fixture(autouse=True) +def ai(monkeypatch) -> FakeAi: + fake = FakeAi() + monkeypatch.setattr(agent_scan, "run_ai_step", fake) + return fake + + +def _write(folder: Path, files: dict[str, str]) -> Path: + for relative, content in files.items(): + path = folder / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return folder + + +def _job(kind: str = "github", **source: object) -> HarnessJob: + return HarnessJob.model_validate(_job_data(kind, **source)) + + +def _job_data(kind: str = "github", **source: object) -> dict: + sources = { + "github": { + "kind": "github", + "visibility": "public", + "repository": "acme/support-bot", + "commit_sha": COMMIT, + }, + "archive": {"kind": "archive", "archive_artifact_id": "upload-1"}, + } + return { + "job_id": "job", + "run_id": "run", + "execution": "hosted", + "source": {**sources[kind], **source}, + "runtime": {"isolation": "dedicated_vm"}, + "agent": {"connector": "auto"}, + } + + +def _scan(job: HarnessJob, source: Path, output: Path) -> int: + return asyncio.run(_work(job, source, output)) + + +def _read_mode(kind: str, source: Path, output: Path) -> int: + """The whole mode, through the helper that runs every reading, from a job file.""" + job_path = output.parent / f"job-{output.name}.json" + job_path.write_text(json.dumps(_job_data(kind)), encoding="utf-8") + return reading_entry.run_reading_mode( + FIND_AGENTS, job_path=job_path, source=source, output=output + ) + + +def _read(output: Path, name: str) -> dict: + return json.loads((output / name).read_text(encoding="utf-8")) + + +def _agents(output: Path) -> dict: + document = _read(output, "agents.json") + assert wire_problems(document) == [] + if document["error"] is not None: + assert document["error"]["message"].strip() + return document + + +def _entries(document: dict) -> list[str]: + return [agent["entry"] for agent in document["agents"]] + + +def _candidate(entry: str, subfolder: str, *, side: bool = True) -> Candidate: + return Candidate( + entry=entry, + subfolder=subfolder, + sign="client", + side=side, + builds=(), + build_may_stop="", + partial=False, + env_names=(), + tools=0, + prompt_files=0, + evidence=(), + ) + + +def _walk(**changes: object) -> WalkResult: + values: dict = { + "agents": (), + "more_than_fifty": False, + "stopped": "", + "unread_files": 0, + "source_outside_side": True, + "candidate_outside_side": False, + "script_outside_side": False, + "only_weak_outside_side": False, + "paths_left_out": 0, + "upload_others": 0, + "upload_others_side": 0, + } + values.update(changes) + return WalkResult(**values) + + +def _tree(folder: Path, leave_out: Path) -> dict[str, tuple[int, int]]: + seen = {} + for directory, directories, files in os.walk(folder, followlinks=False): + for name in directories + files: + path = Path(directory, name) + if path == leave_out or leave_out in path.parents: + continue + status = os.lstat(path) + seen[str(path)] = (status.st_size, status.st_mtime_ns) + return seen + + +def test_a_remote_source_or_a_subfolder_is_refused() -> None: + github = _job() + jobs = [ + github.model_copy( + update={ + "source": RepositorySource.model_construct( + kind=SourceKind.REMOTE, endpoint="https://agent.example" + ) + } + ), + github.model_copy( + update={"source": RepositorySource.model_construct(kind=SourceKind.PROVIDER)} + ), + _job(subfolder="services/bot"), + ] + + for job in jobs: + refused = _check_job(job) + assert refused is not None + assert refused.code == "scan_job_invalid" + assert refused.domain is FailureDomain.PLATFORM_SYNC + assert refused.stage is HarnessStage.UNDERSTANDING_AGENT + assert refused.retryable is False + assert refused.message.strip() + + +def test_the_scan_job_passes_the_kits_job_check() -> None: + # Built through HarnessJob's own validation, as the helper parses the job. + for job in (_job(), _job("archive"), _job(subfolder="")): + assert _check_job(job) is None + + +def test_the_mode_settings(monkeypatch) -> None: + assert FIND_AGENTS.name == "find_agents" + assert FIND_AGENTS.check_job is _check_job + assert FIND_AGENTS.work is _work + assert inspect.iscoroutinefunction(FIND_AGENTS.work) + # The environment hasn't changed since import, so this is the clock the mode was built with. + clock = agent_scan._seconds("ALK_SCAN_MAX_SECONDS", 1200.0) + assert FIND_AGENTS.time_limit_seconds == clock + 600 + assert FIND_AGENTS.timed_out_code == "scan_hard_limit_reached" + assert FIND_AGENTS.timed_out_retryable is False + assert FIND_AGENTS.crashed_code == "scan_crashed" + + for name, default in (("ALK_SCAN_MAX_SECONDS", 1200.0), ("ALK_SCAN_AI_MAX_SECONDS", 600.0)): + monkeypatch.delenv(name, raising=False) + assert agent_scan._seconds(name, default) == default + for raw in ("", "abc", "-5", "0", "inf", "nan"): + monkeypatch.setenv(name, raw) + assert agent_scan._seconds(name, default) == default + monkeypatch.setenv(name, "900") + assert agent_scan._seconds(name, default) == 900.0 + + +def test_a_missing_or_empty_source_is_a_retryable_failure(tmp_path, ai) -> None: + a_file = _write(tmp_path / "files", {"bot.py": GRAPH}) / "bot.py" + empty = tmp_path / "empty" + (empty / "sub").mkdir(parents=True) + linked = tmp_path / "linked" + linked.mkdir() + (linked / "bot.py").symlink_to(a_file) + + for index, source in enumerate((tmp_path / "missing", a_file, empty, linked)): + output = tmp_path / f"reading-{index}" + + assert _scan(_job(), source, output) == 1 + + failure = _read(output, "failure.json") + assert failure["code"] == "scan_source_missing" + assert failure["domain"] == "platform_sync" + assert failure["stage"] == "understanding_agent" + assert failure["retryable"] is True + assert not (output / "agents.json").exists() + assert ai.calls == [] + + +def test_none_when_nothing_looks_like_an_agent(tmp_path, ai) -> None: + quiet = _write(tmp_path / "quiet", {"README.md": "# notes\n"}) + + assert _scan(_job(), quiet, tmp_path / "out-quiet") == 0 + + document = _agents(tmp_path / "out-quiet") + assert (document["status"], document["agents"], document["truncated"]) == ("none", [], False) + assert ai.calls == [] + + plain = _write(tmp_path / "plain", {"app/main.py": PLAIN}) + ai.outcome = _outcome("empty") + + assert _scan(_job(), plain, tmp_path / "out-plain") == 0 + + document = _agents(tmp_path / "out-plain") + assert (document["status"], document["agents"]) == ("none", []) + assert len(ai.calls) == 1 + + +def test_the_walk_limit_before_any_agent_is_an_error(tmp_path, monkeypatch, ai) -> None: + source = _write(tmp_path / "source", {"app/main.py": PLAIN}) + monkeypatch.setattr( + agent_scan, "walk_repo", lambda *args, **kwargs: _walk(stopped="limit", unread_files=2) + ) + + assert _scan(_job(), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert document["status"] == "error" + assert document["error"]["code"] == "scan_limit_reached" + assert document["truncated"] is True + assert ai.calls == [] + + +def test_the_clock_ends_in_a_file(tmp_path, monkeypatch, ai) -> None: + source = _write(tmp_path / "source", {"a/graph.py": GRAPH}) + monkeypatch.setattr(agent_scan, "CLOCK", -1.0) + + assert _scan(_job(), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert document["status"] == "error" + assert document["error"]["code"] == "scan_timed_out" + assert ai.calls == [] + + +def test_a_clock_out_during_the_checks_lists_the_agents_unchecked(tmp_path, checker) -> None: + source = _write(tmp_path / "source", {"a/graph.py": GRAPH, "b/graph.py": GRAPH}) + checker.fail_at = 1 + + assert _scan(_job(), source, tmp_path / "first") == 0 + + document = _agents(tmp_path / "first") + assert (document["status"], _entries(document)) == ("found", ["a/graph.py", "b/graph.py"]) + assert document["truncated"] is True + assert {agent["framework"]["name"] for agent in document["agents"]} == {"unknown"} + assert {agent["confidence"] for agent in document["agents"]} == {"low"} + + checker.calls.clear() + checker.fail_at = 2 + + assert _scan(_job(), source, tmp_path / "second") == 0 + + document = _agents(tmp_path / "second") + assert (document["status"], _entries(document)) == ("found", ["a/graph.py", "b/graph.py"]) + assert document["agents"][1]["framework"]["name"] == "unknown" + assert document["truncated"] is True + + +def test_an_upload_gives_one_agent_at_the_root(tmp_path) -> None: + source = _write(tmp_path / "source", {"a/graph.py": GRAPH, "b/graph.py": GRAPH}) + + assert _scan(_job("archive", commit_sha=COMMIT), source, tmp_path / "upload") == 0 + + document = _agents(tmp_path / "upload") + assert document["status"] == "found" + assert [agent["subfolder"] for agent in document["agents"]] == [""] + assert document["truncated"] is True + assert document["commit_sha"] is None + + assert _scan(_job(), source, tmp_path / "github") == 0 + + document = _agents(tmp_path / "github") + assert sorted(_entries(document)) == ["a/graph.py", "b/graph.py"] + assert document["commit_sha"] == COMMIT.lower() + + +def test_an_uploads_ai_row_is_at_the_root(tmp_path, ai) -> None: + source = _write(tmp_path / "source", {"svc/main.py": PLAIN, "svc/bot.py": PLAIN}) + ai.outcome = _outcome("pairs", [("svc", "svc/main.py"), ("svc", "svc/bot.py")]) + + assert _scan(_job("archive"), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert len(ai.calls) == 1 + assert _entries(document) == ["svc/main.py"] + assert document["agents"][0]["subfolder"] == "" + assert document["truncated"] is True + assert document["commit_sha"] is None + + +def test_the_ai_runs_only_when_nothing_is_found(tmp_path, monkeypatch, ai) -> None: + side_only = _write(tmp_path / "side-only", SIDE_ONLY) + + monkeypatch.setattr(agent_scan, "CLOCK", 1000.0) + assert _scan(_job(), side_only, tmp_path / "out-1000") == 0 + assert [call["seconds"] for call in ai.calls] == [600.0] + assert "test_bot" not in ai.calls[0]["hints"] + + ai.calls.clear() + monkeypatch.setattr(agent_scan, "CLOCK", 500.0) + assert _scan(_job(), side_only, tmp_path / "out-500") == 0 + assert len(ai.calls) == 1 + # What's left after the walk, less the 180 s kept for what follows the model. + assert 300.0 < ai.calls[0]["seconds"] <= 320.0 + + ai.calls.clear() + monkeypatch.setattr(agent_scan, "CLOCK", 300.0) + assert _scan(_job(), side_only, tmp_path / "out-300") == 0 + assert ai.calls == [] + + monkeypatch.setattr(agent_scan, "CLOCK", 1200.0) + found = _write(tmp_path / "found", {**SIDE_ONLY, "bot/graph.py": GRAPH}) + assert _scan(_job(), found, tmp_path / "out-found") == 0 + assert ai.calls == [] + assert "bot/graph.py" in _entries(_agents(tmp_path / "out-found")) + + only_side = _write(tmp_path / "only-side", {"tests/test_bot.py": CLIENT}) + assert _scan(_job(), only_side, tmp_path / "out-only-side") == 0 + assert ai.calls == [] + + +def test_hints_carry_no_path(tmp_path, monkeypatch, ai) -> None: + walk = _walk( + agents=(_candidate("examples/zebra_place/quokka_bot.py", "examples/zebra_place"),), + unread_files=3, + ) + + hints = _hints(walk) + + for word in ("zebra", "quokka", "examples", "/", ".py"): + assert word not in hints + assert "1" in hints and "3" in hints + + source = _write(tmp_path / "source", {"app/main.py": PLAIN}) + monkeypatch.setattr(agent_scan, "walk_repo", lambda *args, **kwargs: walk) + + assert _scan(_job(), source, tmp_path / "out") == 0 + + assert ai.calls[0]["hints"] == hints + + +def test_side_rows_and_ai_rows_together_keep_fifty(tmp_path, checker, ai) -> None: + files = {f"tests/test_bot_{index:02d}.py": CLIENT for index in range(30)} + files.update({f"ai/bot_{index:02d}.py": PLAIN for index in range(30)}) + source = _write(tmp_path / "source", files) + # The last pair repeats a walk entry under a different folder. + ai.outcome = _outcome( + "pairs", + [("ai", f"ai/bot_{index:02d}.py") for index in range(30)] + [("", "tests/test_bot_00.py")], + ) + + assert _scan(_job(), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert len(document["agents"]) == 50 + assert document["truncated"] is True + repeated = [agent for agent in document["agents"] if agent["entry"] == "tests/test_bot_00.py"] + assert [agent["subfolder"] for agent in repeated] == ["tests"] + assert AI_SENTENCE not in repeated[0]["evidence"] + # 30 walk rows and 30 new AI rows; the repeated entry is never checked. + assert len(checker.calls) == 60 + + +def test_a_walk_row_beats_an_ai_row_for_the_same_entry(tmp_path, checker, ai) -> None: + source = _write(tmp_path / "source", SIDE_ONLY) + ai.outcome = _outcome("pairs", [("", "tests/test_bot.py")]) + + assert _scan(_job(), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert len(ai.calls) == 1 + assert _entries(document) == ["tests/test_bot.py"] + assert document["agents"][0]["subfolder"] == "tests" + assert AI_SENTENCE not in document["agents"][0]["evidence"] + assert checker.calls == ["tests"] + + +def test_a_crash_after_the_ai_step_keeps_what_was_paid_for( + tmp_path, monkeypatch, checker, ai +) -> None: + source = _write(tmp_path / "source", SIDE_ONLY) + ai.outcome = _outcome("pairs", [("app", "app/main.py")]) + checker.fail_at = 2 + checker.error = FolderCheckerBroken("the child ended") + + assert _scan(_job(), source, tmp_path / "broken-check") == 0 + + document = _agents(tmp_path / "broken-check") + assert document["status"] == "found" + assert sorted(_entries(document)) == ["app/main.py", "tests/test_bot.py"] + assert document["truncated"] is True + paid = next(agent for agent in document["agents"] if agent["entry"] == "app/main.py") + assert paid["framework"]["name"] == "unknown" + assert not (tmp_path / "broken-check" / "failure.json").exists() + + checker.fail_at, checker.calls = None, [] + + def out_of_memory(*args, **kwargs): + raise MemoryError() + + with monkeypatch.context() as patched: + patched.setattr(agent_scan, "describe_entry", out_of_memory) + + assert _scan(_job(), source, tmp_path / "describe") == 0 + + document = _agents(tmp_path / "describe") + assert (document["status"], _entries(document)) == ("found", ["tests/test_bot.py"]) + assert document["truncated"] is True + + real_write = agent_scan.write_agents_json + + def refuse_found(output, body): + if b'"status":"found"' in body: + raise OSError("disk full") + return real_write(output, body) + + checker.fail_at, checker.calls = 2, [] + with monkeypatch.context() as patched: + patched.setattr(agent_scan, "write_agents_json", refuse_found) + + assert _scan(_job(), source, tmp_path / "write-fails") == 0 + + document = _agents(tmp_path / "write-fails") + assert document["status"] == "error" + assert document["error"]["code"] == "scan_failed_after_ai" + assert document["agents"] == [] + assert not (tmp_path / "write-fails" / "failure.json").exists() + + def refuse_all(output, body): + raise OSError("disk full") + + checker.calls = [] + with monkeypatch.context() as patched: + patched.setattr(agent_scan, "write_agents_json", refuse_all) + + with pytest.raises(FolderCheckerBroken): + _scan(_job(), source, tmp_path / "nothing-writes") + + assert (tmp_path / "nothing-writes" / "cost.json").exists() + + +def test_an_upload_prefers_a_non_side_ai_row_to_a_side_walk_row(tmp_path, ai) -> None: + source = _write(tmp_path / "source", SIDE_ONLY) + ai.outcome = _outcome("pairs", [("app", "app/main.py")]) + + assert _scan(_job("archive"), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert _entries(document) == ["app/main.py"] + assert document["agents"][0]["subfolder"] == "" + assert document["truncated"] is True + + +def test_a_4xx_with_walk_rows_keeps_them(tmp_path, ai) -> None: + ai.outcome = _outcome("model_error", retryable=False) + with_rows = _write(tmp_path / "with-rows", SIDE_ONLY) + + assert _scan(_job(), with_rows, tmp_path / "out-rows") == 0 + + document = _agents(tmp_path / "out-rows") + assert (document["status"], _entries(document)) == ("found", ["tests/test_bot.py"]) + assert document["truncated"] is True + assert not (tmp_path / "out-rows" / "failure.json").exists() + + without = _write(tmp_path / "without", {"app/main.py": PLAIN}) + + assert _scan(_job(), without, tmp_path / "out-none") == 78 + + failure = _read(tmp_path / "out-none", "failure.json") + assert (failure["code"], failure["retryable"]) == ("scan_model_error", False) + assert not (tmp_path / "out-none" / "agents.json").exists() + + +def test_an_ai_row_the_clock_leaves_unchecked_is_still_written(tmp_path, checker, ai) -> None: + source = _write(tmp_path / "source", SIDE_ONLY) + ai.outcome = _outcome("pairs", [("app", "app/main.py")]) + checker.fail_at = 2 + + assert _scan(_job(), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + paid = next(agent for agent in document["agents"] if agent["entry"] == "app/main.py") + assert paid["framework"]["name"] == "unknown" + assert paid["confidence"] == "low" + assert document["truncated"] is True + + +def test_cost_is_written_on_every_return_with_complete(tmp_path, monkeypatch, ai) -> None: + # Driven without the helper, so only the work's own `finally` can have written these. + assert _scan(_job(), tmp_path / "missing", tmp_path / "missing-out") == 1 + assert _read(tmp_path / "missing-out", "cost.json")["complete"] is True + + found = _write(tmp_path / "found", {"a/graph.py": GRAPH}) + assert _scan(_job(), found, tmp_path / "found-out") == 0 + assert _read(tmp_path / "found-out", "cost.json")["complete"] is True + + side_only = _write(tmp_path / "side-only", SIDE_ONLY) + during: list[bool] = [] + ai.during = lambda: during.append(_read(tmp_path / "turn-out", "cost.json")["complete"]) + ai.outcome = _outcome("pairs", [("app", "app/main.py")], complete=False) + + assert _scan(_job(), side_only, tmp_path / "turn-out") == 0 + + assert during == [False] + assert _read(tmp_path / "turn-out", "cost.json")["complete"] is False + + ai.during = None + ai.outcome = _outcome("forbidden") + assert _scan(_job(), side_only, tmp_path / "forbidden-out") == 78 + assert _read(tmp_path / "forbidden-out", "cost.json")["complete"] is True + + def crash(*args, **kwargs): + raise RuntimeError("walk broke") + + monkeypatch.setattr(agent_scan, "walk_repo", crash) + with pytest.raises(RuntimeError): + _scan(_job(), found, tmp_path / "crash-out") + assert (tmp_path / "crash-out" / "cost.json").exists() + + +def test_a_memory_error_ends_in_a_file(tmp_path, monkeypatch, checker, ai) -> None: + source = _write(tmp_path / "source", SIDE_ONLY) + + def out_of_memory(*args, **kwargs): + raise MemoryError() + + monkeypatch.setattr(agent_scan, "walk_repo", out_of_memory) + + assert _scan(_job(), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert document["status"] == "error" + assert document["error"]["code"] == "scan_out_of_memory" + assert checker.calls == [] + assert ai.calls == [] + + +def test_a_broken_checker_before_the_ai_is_a_crash(tmp_path, checker, ai) -> None: + checker.fail_at = 1 + checker.error = FolderCheckerBroken("the child could not start") + + for name, files in (("found", {"a/graph.py": GRAPH}), ("side-only", SIDE_ONLY)): + output = tmp_path / f"out-{name}" + + assert _scan(_job(), _write(tmp_path / name, files), output) == 1 + + failure = _read(output, "failure.json") + assert (failure["code"], failure["domain"], failure["retryable"]) == ( + "scan_crashed", + "infrastructure", + True, + ) + assert not (output / "agents.json").exists() + assert ai.calls == [] + + +@pytest.mark.parametrize( + ("kind", "retryable", "code", "written"), + [ + ("model_error", True, 1, ("failure", "scan_model_error", True)), + ("model_error", False, 78, ("failure", "scan_model_error", False)), + ("forbidden", False, 78, ("failure", "scan_model_forbidden", False)), + ("no_answer", False, 0, ("error", "scan_ai_no_answer")), + ("timed_out", False, 0, ("error", "scan_timed_out")), + ("empty", False, 0, ("none",)), + ], +) +def test_each_ai_outcome_gives_its_exit_code(tmp_path, ai, kind, retryable, code, written) -> None: + source = _write(tmp_path / "source", {"app/main.py": PLAIN}) + ai.outcome = _outcome(kind, retryable=retryable) + output = tmp_path / "out" + + assert _scan(_job(), source, output) == code + + assert len(ai.calls) == 1 + if written[0] == "failure": + failure = _read(output, "failure.json") + assert (failure["code"], failure["retryable"]) == written[1:] + assert failure["domain"] == "infrastructure" + assert not (output / "agents.json").exists() + else: + document = _agents(output) + assert document["status"] == written[0] + if written[0] == "error": + assert document["error"]["code"] == written[1] + assert not (output / "failure.json").exists() + + +def test_every_failure_goes_through_the_helper(tmp_path, monkeypatch, checker, ai) -> None: + ran = tmp_path / "ran" + source = _write( + tmp_path / "source", + {**SIDE_ONLY, "app/main.py": PLAIN + f"open({str(ran)!r}, 'w').close()\n"}, + ) + plain = _write(tmp_path / "plain", {"app/main.py": PLAIN}) + written = [] + real = agent_scan.write_reading_failure + + def recording(failure, output): + written.append(failure) + return real(failure, output) + + monkeypatch.setattr(agent_scan, "write_reading_failure", recording) + + def invalid(**kwargs): + raise ScanResultInvalid("agents.too_many") + + def broken_checker(): + checker.fail_at, checker.error = 1, FolderCheckerBroken("no child") + + def bad_document(patched): + patched.setattr(agent_scan, "finish", invalid) + + def bad_document_after_ai(patched): + ai.outcome = _outcome("pairs", [("app", "app/main.py")]) + patched.setattr(agent_scan, "finish", invalid) + + cases = [ + ("missing", tmp_path / "missing", lambda patched: None, 1, "scan_source_missing"), + ("broken", source, lambda patched: broken_checker(), 1, "scan_crashed"), + ( + "retryable", + plain, + lambda patched: setattr(ai, "outcome", _outcome("model_error", retryable=True)), + 1, + "scan_model_error", + ), + ( + "refused", + plain, + lambda patched: setattr(ai, "outcome", _outcome("model_error")), + 78, + "scan_model_error", + ), + ( + "forbidden", + plain, + lambda patched: setattr(ai, "outcome", _outcome("forbidden")), + 78, + "scan_model_forbidden", + ), + ("invalid", source, bad_document, 78, "scan_result_invalid"), + ("invalid-after-ai", source, bad_document_after_ai, 78, "scan_result_invalid"), + ] + for name, scanned, arrange, code, failure_code in cases: + checker.fail_at, checker.calls = None, [] + ai.outcome = _outcome("empty") + output = tmp_path / f"out-{name}" + before = _tree(tmp_path, output) + with monkeypatch.context() as patched: + arrange(patched) + + assert _scan(_job(), scanned, output) == code + + assert written[-1].code == failure_code + assert written[-1].stage is HarnessStage.UNDERSTANDING_AGENT + assert written[-1].message.strip() + failure = _read(output, "failure.json") + assert (failure["code"], failure["stage"], failure["message"]) == ( + failure_code, + "understanding_agent", + written[-1].message, + ) + assert not (output / "agents.json").exists() + # Nothing is written outside the output folder, and the source never runs. + assert _tree(tmp_path, output) == before + + # A scan that ends in agents.json writes it into the output folder and nowhere else. + checker.fail_at, checker.calls = None, [] + ai.outcome = _outcome("empty") + output = tmp_path / "out-found" + before = _tree(tmp_path, output) + + assert _scan(_job(), source, output) == 0 + + assert _agents(output)["status"] == "found" + assert _tree(tmp_path, output) == before + assert len(written) == len(cases) + assert not ran.exists() + + +def test_a_refused_pair_in_the_kept_submit_sets_truncated(tmp_path, ai) -> None: + source = _write(tmp_path / "source", {"app/main.py": PLAIN}) + + for refused, truncated in ((0, False), (1, True)): + ai.outcome = _outcome("pairs", [("app", "app/main.py")], refused=refused) + output = tmp_path / f"out-{refused}" + + assert _scan(_job(), source, output) == 0 + + document = _agents(output) + assert _entries(document) == ["app/main.py"] + assert document["truncated"] is truncated + + +def test_a_crash_after_the_ai_step_logs_its_class_name_only( + tmp_path, monkeypatch, caplog, ai +) -> None: + source = _write(tmp_path / "source", SIDE_ONLY) + ai.outcome = _outcome("pairs", [("app", "app/main.py")]) + sentinel = "/repo/app/.env FAKE_VALUE_123" + + def crash(*args, **kwargs): + raise RuntimeError(sentinel) + + monkeypatch.setattr(agent_scan, "describe_entry", crash) + with caplog.at_level(logging.DEBUG, logger=agent_scan.logger.name): + assert _scan(_job(), source, tmp_path / "out") == 0 + + logged = [record.getMessage() for record in caplog.records if record.name == agent_scan.logger.name] + assert any("RuntimeError" in line for line in logged) + assert not any("FAKE_VALUE" in line or "/repo/" in line for line in logged) + assert _agents(tmp_path / "out")["status"] == "found" + + +def test_an_uploads_crash_after_the_ai_step_still_writes_one_row(tmp_path, checker, ai) -> None: + source = _write(tmp_path / "source", {**SIDE_ONLY, "svc/bot.py": PLAIN}) + ai.outcome = _outcome("pairs", [("app", "app/main.py"), ("svc", "svc/bot.py")]) + # The walk's one side row is checked first; the AI rows' check then breaks. + checker.fail_at = 2 + checker.error = FolderCheckerBroken("the child ended") + + assert _scan(_job("archive"), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert len(document["agents"]) == 1 + assert _entries(document) == ["app/main.py"] + assert document["agents"][0]["subfolder"] == "" + assert document["truncated"] is True + assert document["commit_sha"] is None + + +def test_an_upload_with_only_side_rows_keeps_the_walks_row(tmp_path, ai) -> None: + source = _write(tmp_path / "source", {**SIDE_ONLY, "tests/test_other.py": PLAIN}) + ai.outcome = _outcome("pairs", [("tests", "tests/test_other.py")]) + + assert _scan(_job("archive"), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert len(ai.calls) == 1 + assert _entries(document) == ["tests/test_bot.py"] + assert document["agents"][0]["subfolder"] == "" + assert document["truncated"] is True + + +def test_hints_count_an_uploads_other_side_files(tmp_path) -> None: + walk = _walk( + agents=(_candidate("tests/test_bot.py", ""),), + upload_others=2, + upload_others_side=2, + unread_files=0, + ) + + assert "agent-like files inside them: 3;" in _hints(walk) + + +def _beside_a_script_agent(helper: str) -> dict[str, str]: + return {"web/src/agent.ts": SCRIPT_AGENT, "backend/judge.py": HELPERS[helper]} + + +@pytest.mark.parametrize("helper", sorted(HELPERS)) +def test_a_script_agent_beside_a_python_model_helper_is_asked_for(tmp_path, ai, helper) -> None: + source = _write(tmp_path / "source", _beside_a_script_agent(helper)) + ai.outcome = _outcome("pairs", [("web", "web/src/agent.ts")]) + output = tmp_path / "out" + + assert _read_mode("github", source, output) == 0 + + document = _agents(output) + assert len(ai.calls) == 1 + assert document["status"] == "found" + assert _entries(document) == ["backend/judge.py", "web/src/agent.ts"] + assert _read(output, "cost.json")["complete"] is True + + +# Every suffix spelled out, so a suffix dropped from the walk's list fails its own case. +@pytest.mark.parametrize("suffix", [".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs"]) +def test_an_agent_in_any_script_suffix_beside_a_helper_is_asked_for(tmp_path, ai, suffix) -> None: + entry = f"web/agent{suffix}" + source = _write(tmp_path / "source", {entry: SCRIPT_AGENT, "backend/judge.py": MODEL_HELPER}) + ai.outcome = _outcome("pairs", [("web", entry)]) + output = tmp_path / "out" + + assert _read_mode("github", source, output) == 0 + + assert len(ai.calls) == 1 + assert _entries(_agents(output)) == ["backend/judge.py", entry] + + +def test_built_vendored_and_config_scripts_do_not_ask_for_the_model(tmp_path, ai) -> None: + scripts = { + "tailwind.config.js": False, + "eslint.config.mjs": False, + "vite.config.ts": False, + "static/js/jquery.min.js": False, + "web/app.min.js": False, + "frontend/dist/assets/index-3f9a.js": False, + "web/build/main.js": False, + "web/vendor/lib.js": False, + "types/env.d.ts": False, + "types/env.d.mts": False, + "web/src/agent.ts": True, + } + for script, asks in scripts.items(): + shape = script.replace("/", "_") + source = _write(tmp_path / shape, {"app/main.py": CLIENT, script: "export {};\n"}) + output = tmp_path / f"out-{shape}" + ai.calls.clear() + + assert _read_mode("github", source, output) == 0, script + + assert len(ai.calls) == (1 if asks else 0), script + assert _entries(_agents(output)) == ["app/main.py"], script + + +def test_the_model_is_not_asked_past_a_helper_without_a_script_agent_beside_it( + tmp_path, ai +) -> None: + shapes = { + "no_script": ({"backend/judge.py": MODEL_HELPER}, "backend/judge.py"), + "beside_a_framework_agent": ( + {"web/src/agent.ts": SCRIPT_AGENT, "bot/graph.py": GRAPH}, + "bot/graph.py", + ), + "script_only_in_a_side_folder": ( + {"examples/web/agent.ts": SCRIPT_AGENT, "backend/judge.py": MODEL_HELPER}, + "backend/judge.py", + ), + } + for shape, (files, entry) in shapes.items(): + source = _write(tmp_path / shape, files) + output = tmp_path / f"out-{shape}" + + assert _read_mode("github", source, output) == 0, shape + + document = _agents(output) + assert ai.calls == [], shape + assert _entries(document) == [entry], shape + assert _read(output, "cost.json")["complete"] is True, shape + + +@pytest.mark.parametrize("helper", sorted(HELPERS)) +def test_an_upload_prefers_the_models_row_to_a_python_model_helper(tmp_path, ai, helper) -> None: + source = _write(tmp_path / "source", _beside_a_script_agent(helper)) + ai.outcome = _outcome("pairs", [("web", "web/src/agent.ts")]) + output = tmp_path / "out" + + assert _read_mode("archive", source, output) == 0 + + document = _agents(output) + assert len(ai.calls) == 1 + assert _entries(document) == ["web/src/agent.ts"] + assert document["agents"][0]["subfolder"] == "" + assert document["truncated"] is True + + +def test_an_uploads_model_row_in_a_side_folder_loses_to_a_python_client(tmp_path, ai) -> None: + source = _write( + tmp_path / "source", + {**_beside_a_script_agent("client"), "examples/agent.ts": SCRIPT_AGENT}, + ) + ai.outcome = _outcome("pairs", [("examples", "examples/agent.ts")]) + output = tmp_path / "out" + + assert _read_mode("archive", source, output) == 0 + + document = _agents(output) + assert len(ai.calls) == 1 + assert _entries(document) == ["backend/judge.py"] + assert document["truncated"] is True + + +@pytest.mark.parametrize("helper", sorted(HELPERS)) +def test_hints_count_the_model_helpers_outside_side_folders(tmp_path, ai, helper) -> None: + source = _write( + tmp_path / "source", {**_beside_a_script_agent(helper), "tests/test_bot.py": CLIENT} + ) + + assert _read_mode("github", source, tmp_path / "out") == 0 + + hints = ai.calls[0]["hints"] + assert "agent-like files inside them: 1;" in hints + assert "files outside them that only call a model: 1;" in hints + for word in ("judge", "backend", "web", "/", ".py"): + assert word not in hints + + +def test_an_uploads_hints_count_each_unlisted_file_where_it_sits(tmp_path, ai) -> None: + files = { + "a/one.py": CLIENT, + "b/two.py": CLIENT, + "c/three.py": MODEL_HELPER, + "tests/test_four.py": CLIENT, + "web/agent.ts": SCRIPT_AGENT, + } + source = _write(tmp_path / "source", files) + + assert _read_mode("archive", source, tmp_path / "out") == 0 + + hints = ai.calls[0]["hints"] + assert "agent-like files inside them: 1;" in hints + assert "files outside them that only call a model: 3;" in hints + + +def test_the_models_rows_are_kept_beside_fifty_helpers(tmp_path, checker, ai) -> None: + fifty = {f"p{index:02d}/main.py": CLIENT for index in range(50)} + forty_seven = {f"p{index:02d}/main.py": CLIENT for index in range(47)} + scripts = {f"web{index}/agent.ts": SCRIPT_AGENT for index in range(5)} + # Each helper in a folder of its own, so every walk row is `medium` and every model row + # `low`: by confidence alone, the cap would drop the model's rows. + shapes = { + "fifty_and_one": ( + {**fifty, "web0/agent.ts": SCRIPT_AGENT}, + [("web0", "web0/agent.ts")], + ["web0/agent.ts"], + ), + "forty_seven_and_five": ( + {**forty_seven, **scripts}, + [(f"web{index}", f"web{index}/agent.ts") for index in range(5)], + sorted(scripts), + ), + # A model row inside a side folder keeps its place behind every walk row. + "fifty_and_one_in_a_side_folder": ( + {**fifty, "web0/agent.ts": SCRIPT_AGENT, "examples/agent.ts": SCRIPT_AGENT}, + [("examples", "examples/agent.ts")], + [], + ), + } + for shape, (files, pairs, kept) in shapes.items(): + source = _write(tmp_path / shape, files) + output = tmp_path / f"out-{shape}" + ai.calls.clear() + ai.outcome = _outcome("pairs", pairs) + + assert _read_mode("github", source, output) == 0, shape + + document = _agents(output) + written = _entries(document) + assert len(ai.calls) == 1, shape + assert len(written) == 50, shape + assert [entry for entry in written if not entry.startswith("p")] == kept, shape + assert document["truncated"] is True, shape + + +def test_the_models_row_beside_fifty_helpers_survives_a_crash_after_the_model( + tmp_path, checker, ai +) -> None: + files = {f"p{index:02d}/main.py": CLIENT for index in range(50)} + source = _write(tmp_path / "source", {**files, "web/agent.ts": SCRIPT_AGENT}) + ai.outcome = _outcome("pairs", [("web", "web/agent.ts")]) + # The walk's fifty checks pass; the model row's check crashes. + checker.fail_at, checker.error = 51, RuntimeError("the checker broke") + output = tmp_path / "out" + + assert _read_mode("github", source, output) == 0 + + document = _agents(output) + assert len(ai.calls) == 1 + assert "web/agent.ts" in _entries(document) + assert len(document["agents"]) == 50 + assert document["truncated"] is True + + +def test_a_langchain_chat_model_needs_no_ai_step(tmp_path, ai) -> None: + example = Path(__file__).parents[2] / "examples" / "frameworks" / "langchain_agent.py" + source = tmp_path / "source" + source.mkdir() + shutil.copyfile(example, source / "langchain_agent.py") + + assert _scan(_job(), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert ai.calls == [] + assert document["status"] == "found" + assert _entries(document) == ["langchain_agent.py"] + assert document["agents"][0]["confidence"] == "low" + + +def test_every_hazard_look_gets_the_scans_one_memo_and_its_deadline( + tmp_path, monkeypatch, ai +) -> None: + source = _write(tmp_path / "source", {**SIDE_ONLY, "web/main.py": PLAIN}) + ai.outcome = _outcome("pairs", [("app", "app/main.py"), ("web", "web/main.py")]) + looks: list[tuple[str, float, object]] = [] + real = scan_walk.build_hazard + + # The real defaults, so a call that leaves a keyword out is recorded with the default. + def recording(source, subfolder, *, deadline=math.inf, clock=time.monotonic, hazards=None): + looks.append((subfolder, deadline, hazards)) + return real(source, subfolder, deadline=deadline, clock=clock, hazards=hazards) + + monkeypatch.setattr(scan_walk, "build_hazard", recording) + start = time.monotonic() + + assert _scan(_job(), source, tmp_path / "out") == 0 + + end = time.monotonic() + assert len(ai.calls) == 1 + # The walk's side row first, then each row the model found. + assert [subfolder for subfolder, _, _ in looks] == ["tests", "app", "web"] + memo = looks[0][2] + assert isinstance(memo, dict) + assert all(hazards is memo for _, _, hazards in looks) + deadlines = {deadline for _, deadline, _ in looks} + assert len(deadlines) == 1 + [deadline] = deadlines + assert start + agent_scan.CLOCK <= deadline <= end + agent_scan.CLOCK + + +def test_a_file_with_a_chat_model_and_a_client_is_a_low_model_row( + tmp_path, checker, ai +) -> None: + source = _write( + tmp_path / "source", + { + "bot.py": ( + "from langchain_openai import ChatOpenAI\n" + "from openai import OpenAI\n" + "\n" + "ChatOpenAI()\n" + "OpenAI()\n" + ) + }, + ) + checker.name = "langchain" + + assert _scan(_job(), source, tmp_path / "out") == 0 + + document = _agents(tmp_path / "out") + assert ai.calls == [] + [agent] = document["agents"] + assert agent["confidence"] == "low" + assert "Calls a chat model through LangChain in bot.py, line 4." in agent["evidence"] + assert "Creates a client for openai in bot.py, line 5." in agent["evidence"] + + +def test_an_agent_at_a_path_the_platform_cannot_carry_is_never_reported_as_none( + tmp_path, ai +) -> None: + """The vercel/ai-chatbot run: the model found ``app/(chat)/api/chat/route.ts`` and every + submit was refused, which used to be written as a whole list with no agents.""" + source = _write(tmp_path / "source", {"app/main.py": PLAIN}) + + ai.outcome = _outcome("empty", uncarried=1) + assert _scan(_job(), source, tmp_path / "out-empty") == 0 + document = _agents(tmp_path / "out-empty") + assert document["agents"] == [] + assert document["truncated"] is True + assert document["error"]["code"] == "agent_path_unsupported" + + ai.outcome = _outcome("pairs", [("app", "app/main.py")], uncarried=1) + assert _scan(_job(), source, tmp_path / "out-pairs") == 0 + document = _agents(tmp_path / "out-pairs") + assert _entries(document) == ["app/main.py"] + assert document["truncated"] is True + assert document["error"] is None diff --git a/tests/harness/test_authoring_entrypoint_find_agents.py b/tests/harness/test_authoring_entrypoint_find_agents.py new file mode 100644 index 000000000..709544dc8 --- /dev/null +++ b/tests/harness/test_authoring_entrypoint_find_agents.py @@ -0,0 +1,242 @@ +"""The `--find-agents` switch: it reaches the scan, it can't be given with the reading's switch, +and a repository scanned through it leaves a file the platform's reader accepts.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from fi.alk.harness import agent_scan, reading_entry, spend +from fi.alk.harness.agent_scan import FIND_AGENTS +from fi.alk.harness.authoring_entrypoint import main +from fi.alk.harness.scan_ai import AiOutcome +from fi.alk.harness.scan_document import wire_problems +from fi.alk.harness.scan_folders import FolderWord + +GRAPH = "from langgraph.graph import StateGraph\nStateGraph(dict)\n" +CLIENT = "from openai import OpenAI\nOpenAI()\n" +COMMIT = "CD" * 20 + + +class FakeChecker: + def __enter__(self) -> FakeChecker: + return self + + def __exit__(self, *exc: object) -> None: + return None + + def check(self, source, subfolder, *, deadline, clock=None) -> FolderWord: + return FolderWord({"name": "langgraph", "version": None, "label": None}, finished=True) + + +@pytest.fixture(autouse=True) +def _fresh_state(monkeypatch, tmp_path): + monkeypatch.delenv(spend.JOURNAL_ALIAS, raising=False) + monkeypatch.setattr(spend, "_path", None) + # The machine's own secrets files must not decide what a test sees masked. + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (tmp_path / "no-secrets.json",)) + monkeypatch.setattr(reading_entry, "_secret_values", None) + spend._stages.clear() + yield + spend._stages.clear() + + +@pytest.fixture +def fakes(monkeypatch) -> list[dict]: + """The folder checker and the model step, faked; the model's calls are returned.""" + asked: list[dict] = [] + + async def no_model(source, *, hints, seconds, on_turn_end) -> AiOutcome: + asked.append({"hints": hints}) + return AiOutcome("empty", (), 0, True, False, "fixed words") + + monkeypatch.setattr(agent_scan, "FolderChecker", FakeChecker) + monkeypatch.setattr(agent_scan, "run_ai_step", no_model) + return asked + + +def _write(folder: Path, files: dict[str, str]) -> Path: + for relative, content in files.items(): + path = folder / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return folder + + +_SOURCES = { + "github": { + "kind": "github", + "visibility": "public", + "repository": "acme/support-bot", + "commit_sha": COMMIT, + }, + "archive": {"kind": "archive", "archive_artifact_id": "upload-1"}, +} + + +def _job_file(tmp_path: Path, kind: str = "github") -> Path: + path = tmp_path / f"job-{kind}.json" + path.write_text( + json.dumps( + { + "job_id": "job", + "run_id": "run", + "execution": "hosted", + "source": _SOURCES[kind], + "runtime": {"isolation": "dedicated_vm"}, + "agent": {"connector": "auto"}, + } + ), + encoding="utf-8", + ) + return path + + +def _found(output: Path) -> dict: + document = json.loads((output / "agents.json").read_text(encoding="utf-8")) + assert wire_problems(document) == [] + return document + + +def test_the_switch_reaches_the_scan(tmp_path, monkeypatch) -> None: + calls = [] + + def recording(mode, *, job_path, source, output): + calls.append((mode, job_path, source, output)) + return 0 + + monkeypatch.setattr(reading_entry, "run_reading_mode", recording) + job = _job_file(tmp_path) + + code = main( + [ + str(job), + "--source", + str(tmp_path / "a" / ".." / "source"), + "--output", + str(tmp_path / "a" / ".." / "reading"), + "--find-agents", + ] + ) + + assert code == 0 + assert calls == [ + ( + FIND_AGENTS, + job, + (tmp_path / "source").resolve(), + (tmp_path / "reading").resolve(), + ) + ] + + +def test_the_two_switches_exclude_each_other(tmp_path, monkeypatch) -> None: + calls = [] + monkeypatch.setattr( + reading_entry, "run_reading_mode", lambda mode, **kwargs: calls.append(mode) or 0 + ) + + with pytest.raises(SystemExit) as stopped: + main( + [ + str(_job_file(tmp_path)), + "--source", + str(tmp_path / "source"), + "--output", + str(tmp_path / "reading"), + "--read-agent-only", + "--find-agents", + ] + ) + + assert stopped.value.code == 2 + assert calls == [] + + +def test_a_github_repo_gives_found(tmp_path, fakes) -> None: + source = _write( + tmp_path / "source", + {"graph.py": GRAPH, "pyproject.toml": '[project]\nname = "bot"\n'}, + ) + output = tmp_path / "reading" + + code = main( + [ + str(_job_file(tmp_path)), + "--source", + str(source), + "--output", + str(output), + "--find-agents", + ] + ) + + assert code == 0 + document = _found(output) + assert document["status"] == "found" + assert document["commit_sha"] == COMMIT.lower() + assert [(agent["entry"], agent["subfolder"]) for agent in document["agents"]] == [ + ("graph.py", "") + ] + assert document["agents"][0]["name"] == "support-bot" + assert document["truncated"] is False + assert json.loads((output / "cost.json").read_text(encoding="utf-8"))["complete"] is True + assert not (output / "failure.json").exists() + assert fakes == [] + + +def test_fifty_one_agents_give_fifty_and_truncated(tmp_path, fakes) -> None: + source = _write(tmp_path / "source", {f"bot_{index:02d}.py": CLIENT for index in range(51)}) + output = tmp_path / "reading" + + code = main( + [ + str(_job_file(tmp_path)), + "--source", + str(source), + "--output", + str(output), + "--find-agents", + ] + ) + + assert code == 0 + document = _found(output) + assert document["status"] == "found" + assert len(document["agents"]) == 50 + assert document["truncated"] is True + assert fakes == [] + + +def test_an_upload_with_nothing_agent_like_is_none(tmp_path, fakes) -> None: + # Nothing that looks like code, so no model is asked; then plain code, so the model is + # asked and finds nothing. + shapes = { + "quiet": ({"README.md": "# notes\n"}, 0), + "plain": ({"app/main.py": "def main():\n return 1\n"}, 1), + } + for shape, (files, asked) in shapes.items(): + fakes.clear() + source = _write(tmp_path / shape, files) + output = tmp_path / f"reading-{shape}" + + code = main( + [ + str(_job_file(tmp_path, "archive")), + "--source", + str(source), + "--output", + str(output), + "--find-agents", + ] + ) + + assert code == 0, shape + document = _found(output) + assert (document["status"], document["agents"], document["error"]) == ("none", [], None) + assert document["commit_sha"] is None + assert len(fakes) == asked, shape + assert (output / "cost.json").exists() + assert not (output / "failure.json").exists() diff --git a/tests/harness/test_scan_ai.py b/tests/harness/test_scan_ai.py new file mode 100644 index 000000000..be3cca7bd --- /dev/null +++ b/tests/harness/test_scan_ai.py @@ -0,0 +1,1007 @@ +"""The AI step of the agent scan: one paid turn, what it ends as, and the tools the model reads with. + +No test calls a model. A fake backend plays a script: what the model says, which tools it calls +(through the real handlers), and how its turn ends. Every test that waits on a stage runs under +its own bound, so a missing bound in the code fails the test instead of hanging the run. +""" + +from __future__ import annotations + +import asyncio +import os +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from fi.alk.harness import scan_ai, session, spend +from fi.alk.harness.backends import Call, ModelReply, Say, StageDone, ToolReturned +from fi.alk.harness.config import load_skill + +GEMINI = "gemini-fake-model" +SUBMIT = "mcp__scan__submit_agents" +READ = "mcp__scan_files__read" +GLOB = "mcp__scan_files__glob" +GREP = "mcp__scan_files__grep" +AGENT = ("services/support", "services/support/agent.py") +KEPT = ("call", SUBMIT, {"agents": [{"folder": AGENT[0], "entry": AGENT[1]}]}) +FINISHED = ("done", StageDone(outcome="success", turns=3, cost_usd=0.01, models={GEMINI})) +# ADK's whole message, as the Vertex-Gemini backend passes it on. +CALL_LIMIT_MESSAGE = "Max number of llm calls limit of `40` exceeded" +TEST_BOUND = 15 + +# Read before any test shortens them. +_BOUNDS = { + name: getattr(scan_ai, name) + for name in ( + "OPEN_SECONDS", + "INTERRUPT_SECONDS", + "SPEND_WAIT_SECONDS", + "EXIT_SECONDS", + "TOOL_SECONDS", + "IDLE_SECONDS", + "MAX_TURNS", + ) +} + + +@pytest.fixture(autouse=True) +def _clean_ledger(monkeypatch): + retries = session.STAGE_IDLE_RETRIES + monkeypatch.delenv("ALK_SPEND_JOURNAL", raising=False) + monkeypatch.setattr(spend, "_path", None) + monkeypatch.setenv("ALK_HARNESS_MODEL", GEMINI) + yield + session.STAGE_IDLE_RETRIES = retries + spend._stages.clear() + + +@pytest.fixture +def quick(monkeypatch): + for name, seconds in ( + ("OPEN_SECONDS", 0.3), + ("INTERRUPT_SECONDS", 0.2), + ("SPEND_WAIT_SECONDS", 0.3), + ("CANCEL_SECONDS", 0.2), + ("EXIT_SECONDS", 0.2), + ): + monkeypatch.setattr(scan_ai, name, seconds) + + +def _write(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _repo(tmp_path: Path) -> Path: + source = tmp_path / "repo" + _write(source / AGENT[1], "from openai import OpenAI\nclient = OpenAI()\n") + _write(source / "services/support/pyproject.toml", "[project]\nname = 'support'\n") + _write(source / "main.py", "print('hello')\n") + _write(source / "tests/test_agent.py", "def test_nothing():\n pass\n") + return source + + +def _said(result: dict[str, Any]) -> str: + return "".join(part.get("text", "") for part in result.get("content", [])) + + +class _Backend: + """Sessions that play ``steps`` instead of calling a model, and count what they were asked.""" + + name = "fake" + default_model = GEMINI + + def __init__(self, *steps: tuple, start: str = "", stop: str = "", interrupt: str = ""): + self.steps = steps + self.start = start + self.stop = stop + self.interrupt = interrupt + self.specs: list[Any] = [] + self.sent: list[str] = [] + self.results: list[dict[str, Any]] = [] + self.interrupts = 0 + self.stops = 0 + + def create(self, spec): + self.specs.append(spec) + return _Session(self, spec) + + def can_drive(self, model: str) -> bool: + return True + + +class _Session: + def __init__(self, backend: _Backend, spec) -> None: + self.backend = backend + self.spec = spec + self.interrupted = asyncio.Event() + + async def start(self) -> None: + if self.backend.start == "hang": + await asyncio.Event().wait() + if self.backend.start == "raise": + raise RuntimeError("the provider refused the session") + + async def stop(self) -> None: + self.backend.stops += 1 + if self.backend.stop == "hang": + await asyncio.Event().wait() + + async def send(self, message: str) -> None: + self.backend.sent.append(message) + + async def interrupt(self) -> bool: + self.backend.interrupts += 1 + if self.backend.interrupt == "hang": + await asyncio.Event().wait() + self.interrupted.set() + return True + + async def resume(self, invocation_id: str) -> None: + raise RuntimeError("not supported") + + def _handler(self, qualified: str): + _, server, name = qualified.split("__") + return next( + spec.handler for spec in self.spec.servers[server].tools if spec.name == name + ) + + async def replies(self): + for step in self.backend.steps: + kind = step[0] + if kind == "say": + yield ModelReply(parts=[Say(text=step[1])], model=GEMINI) + elif kind == "call": + call_id = f"call-{len(self.backend.results)}" + yield ModelReply(parts=[Call(id=call_id, name=step[1], arguments=step[2])]) + result = await self._handler(step[1])(step[2]) + self.backend.results.append(result) + yield ToolReturned( + id=call_id, text=_said(result), is_error=bool(result.get("is_error")) + ) + elif kind == "done": + yield step[1] + elif kind == "quiet": + await asyncio.Event().wait() + elif kind == "until_interrupt": + await self.interrupted.wait() + elif kind == "raise": + raise step[1] + + +def _scan(monkeypatch, source: Path, backend: _Backend, *, seconds: float = 5.0, + hints: str = "3 Python files outside side folders", ends: list | None = None): + monkeypatch.setattr(session, "resolve", lambda *_args, **_kwargs: backend) + noted = ends if ends is not None else [] + return asyncio.run( + asyncio.wait_for( + scan_ai.run_ai_step( + source, hints=hints, seconds=seconds, on_turn_end=lambda: noted.append(1) + ), + TEST_BOUND, + ) + ) + + +@dataclass +class _Case: + steps: tuple + kind: str + retryable: bool = False + complete: bool = True + ends: int = 1 + seconds: float = 5.0 + interrupt: str = "" + idle: float | None = None + pairs: tuple = field(default_factory=tuple) + + +def _done(**facts: Any) -> tuple: + facts.setdefault("models", {GEMINI}) + return ("done", StageDone(**facts)) + + +_OUTCOMES = { + "pairs": _Case((KEPT, FINISHED), "pairs", pairs=(AGENT,)), + "pairs_then_a_forbidden_model": _Case( + (KEPT, _done(models={"claude-sonnet-4-6"})), "pairs", pairs=(AGENT,) + ), + "forbidden": _Case((_done(models={"claude-sonnet-4-6"}),), "forbidden"), + "timed_out_quiet": _Case((("quiet",),), "timed_out", complete=False, ends=0, seconds=0.3), + "timed_out_bill_after_the_interrupt": _Case( + (("until_interrupt",), FINISHED), "timed_out", complete=False, seconds=0.3 + ), + "interrupt_never_returns": _Case( + (("quiet",),), "timed_out", complete=False, ends=0, seconds=0.3, interrupt="hang" + ), + "idle_stream": _Case((("quiet",),), "model_error", True, complete=False, ends=0, idle=0.2), + "say_raises": _Case( + (("raise", RuntimeError("stream broke")),), "model_error", True, complete=False, ends=0 + ), + "status_400": _Case( + (_done(outcome="failed", is_error=True, api_error_status=400),), "model_error", False + ), + "status_408": _Case( + (_done(outcome="failed", is_error=True, api_error_status=408),), "model_error", True + ), + "status_409": _Case( + (_done(outcome="failed", is_error=True, api_error_status=409),), "model_error", True + ), + "status_429": _Case( + (_done(outcome="failed", is_error=True, api_error_status=429),), "model_error", True + ), + "status_500": _Case( + (_done(outcome="failed", is_error=True, api_error_status=500),), "model_error", True + ), + "is_error_without_status": _Case( + (_done(outcome="failed", is_error=True, turns=3),), "model_error", True + ), + "is_error_without_status_on_the_last_turn": _Case( + ( + _done( + outcome="failed", + is_error=True, + api_error_status=None, + turns=40, + errors=["The connection was reset by the server."], + ), + ), + "model_error", + True, + ), + "call_limit_words_inside_other_text": _Case( + ( + _done( + outcome="failed", + is_error=True, + api_error_status=None, + turns=7, + errors=[f"Retry failed: {CALL_LIMIT_MESSAGE} (upstream)"], + ), + ), + "model_error", + True, + ), + "call_limit_words_without_a_number": _Case( + ( + _done( + outcome="failed", + is_error=True, + api_error_status=None, + turns=7, + errors=["Max number of llm calls limit of `` exceeded"], + ), + ), + "model_error", + True, + ), + "claude_turn_limit": _Case( + (_done(outcome="error_max_turns", is_error=True, turns=40),), "no_answer" + ), + "gemini_max_turns": _Case( + (_done(outcome="max_turns", is_error=True, turns=12),), "no_answer" + ), + "gemini_call_limit": _Case( + ( + _done( + outcome="failed", + is_error=True, + api_error_status=None, + turns=40, + errors=[CALL_LIMIT_MESSAGE], + ), + ), + "no_answer", + ), + "gemini_call_limit_in_adks_words": _Case( + ( + _done( + outcome="failed", + is_error=True, + api_error_status=None, + turns=7, + errors=[CALL_LIMIT_MESSAGE], + ), + ), + "no_answer", + ), + "empty": _Case((("call", SUBMIT, {"agents": []}), FINISHED), "empty"), + "every_pair_refused": _Case( + (("call", SUBMIT, {"agents": [{"folder": "", "entry": "missing.py"}]}), FINISHED), + "empty", + ), + "no_answer": _Case((("say", "I looked around."), FINISHED), "no_answer"), +} + + +@pytest.mark.parametrize("name", sorted(_OUTCOMES)) +def test_each_ai_outcome_gives_its_result(tmp_path, monkeypatch, quick, name): + case = _OUTCOMES[name] + if case.idle is not None: + monkeypatch.setattr(scan_ai, "IDLE_SECONDS", case.idle) + backend = _Backend(*case.steps, interrupt=case.interrupt) + ends: list = [] + + outcome = _scan(monkeypatch, _repo(tmp_path), backend, seconds=case.seconds, ends=ends) + + assert outcome.kind == case.kind + assert outcome.pairs == case.pairs + assert outcome.complete is case.complete + assert outcome.retryable is case.retryable + assert len(ends) == case.ends + assert session.STAGE_IDLE_RETRIES == 0 + assert len(backend.sent) == 1 + if case.kind == "timed_out": + assert backend.interrupts == 1 + + +def test_a_gemini_call_limit_is_no_answer(tmp_path, monkeypatch): + """The real Vertex-Gemini backend and ADK, with only the model scripted: it never submits.""" + pytest.importorskip("google.adk") + from google.adk.models.base_llm import BaseLlm + from google.adk.models.llm_response import LlmResponse + from google.genai import types + + from fi.alk.harness.backends import vertex_gemini + + calls: list[int] = [] + + class LoopingLlm(BaseLlm): + async def generate_content_async(self, llm_request, stream=False): + calls.append(1) + yield LlmResponse( + content=types.Content( + role="model", + parts=[ + types.Part( + function_call=types.FunctionCall( + name=GLOB, args={"pattern": "*.py"}, id=f"c{len(calls)}" + ) + ) + ], + ), + usage_metadata=types.GenerateContentResponseUsageMetadata( + prompt_token_count=1000, candidates_token_count=10 + ), + ) + + # The backend writes these three itself; set here so they are put back afterwards. + for name, value in ( + ("GOOGLE_CLOUD_PROJECT", "fake-project"), + ("GOOGLE_CLOUD_LOCATION", "global"), + ("GOOGLE_GENAI_USE_VERTEXAI", "TRUE"), + ("ALK_HARNESS_MODEL", "gemini-2.5-flash"), + ): + monkeypatch.setenv(name, value) + monkeypatch.delenv("AGENTCC_API_KEY", raising=False) + monkeypatch.setattr(vertex_gemini, "COMPACT_ABOVE_TOKENS", 0) + monkeypatch.setattr(vertex_gemini, "_llm", lambda model: LoopingLlm(model=model)) + for name, seconds in ( + ("INTERRUPT_SECONDS", 0.2), + ("SPEND_WAIT_SECONDS", 0.3), + ("CANCEL_SECONDS", 0.2), + ("EXIT_SECONDS", 1.0), + ("OPEN_SECONDS", 5.0), + ): + monkeypatch.setattr(scan_ai, name, seconds) + ends: list = [] + + outcome = _scan( + monkeypatch, + _repo(tmp_path), + vertex_gemini.VertexGeminiBackend(), + seconds=8.0, + ends=ends, + ) + + assert outcome.kind == "no_answer" + assert outcome.complete is True + assert len(ends) == 1 + assert 1 <= len(calls) <= scan_ai.MAX_TURNS + 1 + assert "llm calls" not in outcome.message.lower() + + +def test_an_idle_ai_stream_is_not_replayed(tmp_path, monkeypatch, quick): + session.STAGE_IDLE_RETRIES = 1 + monkeypatch.setattr(scan_ai, "IDLE_SECONDS", 0.2) + backend = _Backend(("quiet",)) + + outcome = _scan(monkeypatch, _repo(tmp_path), backend) + + assert outcome.kind == "model_error" + assert outcome.retryable is True + assert len(backend.specs) == 1 + assert len(backend.sent) == 1 + assert session.STAGE_IDLE_RETRIES == 0 + + +def test_pairs_submitted_before_an_idle_stream_are_kept(tmp_path, monkeypatch, quick): + monkeypatch.setattr(scan_ai, "IDLE_SECONDS", 0.2) + backend = _Backend(KEPT, ("quiet",)) + + outcome = _scan(monkeypatch, _repo(tmp_path), backend) + + assert outcome.kind == "pairs" + assert outcome.pairs == (AGENT,) + assert outcome.complete is False + + +# A model that ends without submitting is the turn a reminder would be tempted to follow. +@pytest.mark.parametrize( + ("steps", "kind"), + [((("say", "Looking."), KEPT, FINISHED), "pairs"), ((("say", "Looking."), FINISHED), "no_answer")], + ids=["submitted", "ended_without_submitting"], +) +def test_the_stage_says_exactly_once(tmp_path, monkeypatch, quick, steps, kind): + backend = _Backend(*steps) + ends: list = [] + hints = "3 Python files outside side folders" + + outcome = _scan(monkeypatch, _repo(tmp_path), backend, hints=hints, ends=ends) + + assert outcome.kind == kind + assert len(backend.specs) == 1 + assert backend.sent == [scan_ai._opening(hints)] + assert len(ends) == 1 + + +def test_complete_is_false_after_the_stop_sequence(tmp_path, monkeypatch, quick): + assert _BOUNDS["INTERRUPT_SECONDS"] == 10 + assert _BOUNDS["SPEND_WAIT_SECONDS"] == 60 + backend = _Backend(KEPT, ("until_interrupt",), FINISHED) + ends: list = [] + + outcome = _scan(monkeypatch, _repo(tmp_path), backend, seconds=0.3, ends=ends) + + assert outcome.kind == "pairs" + assert backend.interrupts == 1 + # The bill arrived during the wait, and still the turn may be missing a call in flight. + assert len(ends) == 1 + assert outcome.complete is False + + +@pytest.mark.parametrize("start", ["hang", "raise"]) +def test_a_stage_that_will_not_open_is_a_model_error(tmp_path, monkeypatch, quick, start): + assert _BOUNDS["OPEN_SECONDS"] == 30 + backend = _Backend(KEPT, FINISHED, start=start) + + outcome = _scan(monkeypatch, _repo(tmp_path), backend) + + assert outcome.kind == "model_error" + assert outcome.retryable is True + assert outcome.complete is False + assert backend.sent == [] + + +def test_a_stage_that_will_not_stop_is_left_after_ten_seconds(tmp_path, monkeypatch, quick): + assert _BOUNDS["EXIT_SECONDS"] == 10 + backend = _Backend(KEPT, FINISHED, stop="hang") + + outcome = _scan(monkeypatch, _repo(tmp_path), backend) + + assert outcome.kind == "pairs" + assert outcome.complete is True + assert backend.stops == 1 + + +def test_the_opening_carries_no_path_and_the_outcome_no_model_text(tmp_path, monkeypatch, quick): + source = _repo(tmp_path) + hints = "2 Python files outside side folders; 1 candidate in side folders" + model_text = "MODEL-TEXT services/support/agent.py says sk-fake-123" + assert "/" not in scan_ai._opening("") + + failed = _Backend( + ("say", model_text), + _done(outcome="failed", is_error=True, turns=2, errors=[model_text]), + ) + outcome = _scan(monkeypatch, source, failed, hints=hints) + assert outcome.kind == "model_error" + assert failed.sent == [scan_ai._opening(hints)] + assert failed.sent[0].endswith(hints) + for path in (str(source), str(tmp_path), "services/", "tests/", "agent.py"): + assert path not in failed.sent[0] + + raised = _Backend(("raise", RuntimeError(model_text)),) + raised_outcome = _scan(monkeypatch, source, raised, hints=hints) + assert raised_outcome.message == "the model step failed (RuntimeError)" + + answered = _Backend(("say", model_text), KEPT, FINISHED) + kept = _scan(monkeypatch, source, answered, hints=hints) + assert kept.pairs == (AGENT,) + + for one in (outcome, raised_outcome, kept): + assert "MODEL-TEXT" not in one.message + assert "sk-fake-123" not in one.message + + +def test_the_stage_has_no_built_in_tools(tmp_path, monkeypatch, quick): + source = _repo(tmp_path) + backend = _Backend(FINISHED) + + _scan(monkeypatch, source, backend) + + spec = backend.specs[0] + assert spec.builtins == () + assert spec.ask is None + assert spec.gated is True + assert spec.thinking is False + assert spec.max_turns == 40 + assert spec.idle_timeout_seconds == 90 + assert spec.cwd == str(source) + assert spec.model == GEMINI + assert spec.system_prompt == load_skill("find-agents") + assert set(spec.servers) == {"scan_files", "scan"} + assert sorted(spec.granted()) == sorted([READ, GLOB, GREP, SUBMIT]) + for name in (READ, GLOB, GREP, SUBMIT): + assert name in spec.system_prompt + + +def test_the_skills_description_says_when_the_model_is_asked(): + # The model reads the whole file, its description included. + skill = load_skill("find-agents", preamble=False) + description = yaml.safe_load(skill.split("---\n")[1])["description"] + + assert "came back empty" not in skill + assert "found no agent outside test and example folders" in description + assert "apart from any files that only call a model" in description + assert "TypeScript or JavaScript" in description + assert skill in load_skill("find-agents") + + +# -- the tools --------------------------------------------------------------------------------- + + +def _tools(root: Path) -> dict[str, Any]: + return {spec.name: spec.handler for spec in scan_ai.rooted_file_tools(root).tools} + + +def _call(handler, arguments: Any) -> dict[str, Any]: + return asyncio.run(asyncio.wait_for(handler(arguments), TEST_BOUND)) + + +def _listed(text: str) -> set[str]: + """The paths a glob or grep result names.""" + return {line.split(":", 1)[0] for line in text.splitlines() if line} + + +def test_the_ai_cannot_read_outside_the_source(tmp_path): + outside = _write(tmp_path / "outside/creds.txt", "outside-needle\n") + source = tmp_path / "repo" + _write(source / "inside.py", "inside-needle\n") + os.symlink(outside.parent, source / "out_link") + os.symlink(outside, source / "file_link.py") + tools = _tools(source) + + for path in ( + "../outside/creds.txt", + str(outside), + "out_link/creds.txt", + "file_link.py", + "inside.py/../../outside/creds.txt", + ): + result = _call(tools["read"], {"path": path}) + assert result.get("is_error"), path + assert "outside-needle" not in _said(result) + for path in ("../outside", "/", str(tmp_path), "out_link"): + assert _call(tools["glob"], {"pattern": "*", "path": path}).get("is_error"), path + assert _call(tools["grep"], {"text": "needle", "path": path}).get("is_error"), path + + assert _listed(_said(_call(tools["glob"], {"pattern": "*"}))) == {"inside.py"} + found = _said(_call(tools["grep"], {"text": "needle"})) + assert "outside-needle" not in found + assert _listed(found) == {"inside.py"} + assert "inside-needle" in _said(_call(tools["read"], {"path": str(source / "inside.py")})) + + +_HIDDEN = ( + ".env", + "config/service_account.json", + "config/token.json", + "prod.env", + "app.env.local", + "secrets.toml", + "credentials.yml", + "id_rsa", + "terraform.tfvars", + "secrets/keys.py", +) + + +def test_grep_and_glob_never_show_a_secret_file(tmp_path): + source = tmp_path / "repo" + _write(source / "agent.py", "needle = 1\n") + _write(source / "config/settings.py", "needle = 2\n") + for name in _HIDDEN: + _write(source / name, "needle API_KEY=sk-fake-123\n") + tools = _tools(source) + + for path in (None, ".", "config", str(source), str(source / "config")): + arguments = {} if path is None else {"path": path} + for grep in ({"text": "needle"}, {"text": "NEEDLE", "ignore_case": True}): + found = _said(_call(tools["grep"], {**grep, **arguments})) + assert "sk-fake-123" not in found + assert not _listed(found) & set(_HIDDEN), (path, found) + assert _listed(found) <= {"agent.py", "config/settings.py"} + for pattern in ("*", "**", "*.env", "*.json", "id_*", "*secret*"): + listed = _said(_call(tools["glob"], {"pattern": pattern, **arguments})) + assert not _listed(listed) & set(_HIDDEN), (path, pattern, listed) + assert "agent.py" in _listed(_said(_call(tools["grep"], {"text": "needle"}))) + for name in _HIDDEN: + result = _call(tools["read"], {"path": name}) + assert result.get("is_error"), name + assert "sk-fake-123" not in _said(result) + + +def test_the_tools_skip_fifos_and_big_files_and_hard_links(tmp_path): + source = tmp_path / "repo" + _write(source / "agent.py", "needle\n") + os.mkfifo(source / "pipe.py") + _write(source / "big.py", "needle\n" * 300_000) + _write(source / "linked.py", "needle\n") + os.link(source / "linked.py", source / "also.py") + os.link(_write(tmp_path / "outside.py", "needle\n"), source / "hard.py") + _write(source / "app.min.js", "needle\n") + _write(source / "long.py", "needle" + "x" * 70_000 + "\nneedle\n") + tools = _tools(source) + + assert _listed(_said(_call(tools["glob"], {"pattern": "*"}))) == {"agent.py", "long.py"} + found = _said(_call(tools["grep"], {"text": "needle"})) + assert found.splitlines() == ["agent.py:1:needle", "long.py:2:needle"] + for name in ("pipe.py", "big.py", "linked.py", "also.py", "hard.py"): + assert _call(tools["read"], {"path": name}).get("is_error"), name + assert _call(tools["grep"], {"text": "needle", "path": name}).get("is_error"), name + + +def test_a_slow_tool_call_ends_with_an_error(tmp_path, monkeypatch): + assert _BOUNDS["TOOL_SECONDS"] == 20 + source = _repo(tmp_path) + monkeypatch.setattr(scan_ai, "TOOL_SECONDS", 0.2) + monkeypatch.setattr(scan_ai, "_grep_files", lambda root, arguments: time.sleep(3) or "late") + tools = _tools(source) + + began = time.monotonic() + result = _call(tools["grep"], {"text": "needle"}) + + assert time.monotonic() - began < 2 + assert result.get("is_error") + assert "late" not in _said(result) + + +def test_no_tool_raises(tmp_path, monkeypatch): + source = _repo(tmp_path) + tools = _tools(source) + calls = [ + ("read", {"path": "main\x00.py"}), + ("read", {"path": "main.py", "offset": "x"}), + ("read", {"path": "main.py", "limit": [1]}), + ("read", {"path": "/".join(["a"] * 3_000)}), + ("read", {"path": "a" * 5_000}), + ("read", {"path": 7}), + ("read", {}), + ("read", None), + ("glob", {"pattern": "x" * 300}), + ("glob", {"pattern": None}), + ("glob", {"pattern": "*", "path": "\x00"}), + ("grep", {"text": ""}), + ("grep", {"text": "x" * 201}), + ("grep", {"text": "a", "ignore_case": "yes"}), + ("grep", {"text": "a", "glob": 5}), + ("grep", {"text": "a", "path": "missing/folder"}), + ] + for name, arguments in calls: + assert _call(tools[name], arguments).get("is_error"), (name, arguments) + + def broken(*_args): + raise RuntimeError("MODEL-TEXT from inside a worker") + + for worker in ("_read_lines", "_glob_files", "_grep_files", "_checked_pairs"): + monkeypatch.setattr(scan_ai, worker, broken) + for name, arguments in ( + ("read", {"path": "main.py"}), + ("glob", {"pattern": "*"}), + ("grep", {"text": "a"}), + ): + result = _call(tools[name], arguments) + assert result.get("is_error") + assert _said(result) == scan_ai._FAILED + submit = scan_ai._submit_server(source, scan_ai._Answer()).tools[0].handler + submitted = _call(submit, {"agents": [{"folder": "", "entry": "main.py"}]}) + assert submitted.get("is_error") + assert _said(submitted) == scan_ai._FAILED + + +def test_tool_paths_are_relative(tmp_path): + source = tmp_path / "repo" + _write(source / "pkg/agent.py", "needle\n") + tools = _tools(source) + + for path in (None, "pkg", str(source / "pkg")): + arguments = {} if path is None else {"path": path} + listed = _said(_call(tools["glob"], {"pattern": "*.py", **arguments})) + found = _said(_call(tools["grep"], {"text": "needle", **arguments})) + assert listed == "pkg/agent.py" + assert found == "pkg/agent.py:1:needle" + for text in (listed, found): + assert str(tmp_path) not in text + assert _said(_call(tools["read"], {"path": "pkg/agent.py"})) == "1\tneedle" + + +def test_a_capped_search_says_it_stopped_early(tmp_path): + source = tmp_path / "repo" + for index in range(2_100): + _write(source / f"many/f{index:04}.txt", "") + tools = _tools(source) + + for name, arguments in ( + ("glob", {"pattern": "*.nothing"}), + ("grep", {"text": "nothing-here"}), + ): + said = _said(_call(tools[name], arguments)) + assert said == scan_ai.STOPPED_EARLY, (name, said) + + full = tmp_path / "full" + _write(full / "hits.py", "".join(f"needle {index}\n" for index in range(2_000))) + _write(full / "wide.py", "".join(f"{'y' * 100}\n" for _ in range(400))) + full_tools = _tools(full) + for name, arguments in ( + ("grep", {"text": "needle"}), + ("read", {"path": "wide.py"}), + ): + said = _said(_call(full_tools[name], arguments)) + assert said.endswith(scan_ai.STOPPED_EARLY), name + assert len(said) <= 8_000, name + + +def test_one_long_hit_does_not_hide_the_others(tmp_path): + source = tmp_path / "repo" + _write(source / "a.py", "needle" + "x" * 20_000 + "\n") + for name in ("b.py", "c.py", "d.py", "e.py"): + _write(source / name, "needle\n") + tools = _tools(source) + + found = _said(_call(tools["grep"], {"text": "needle"})) + + assert _listed(found) == {"a.py", "b.py", "c.py", "d.py", "e.py"} + assert scan_ai.STOPPED_EARLY not in found + for line in found.splitlines(): + assert len(line.split(":", 2)[2]) <= 300 + + +# -- the answer -------------------------------------------------------------------------------- + + +def _submit(source: Path, agents: Any) -> tuple[str, Any]: + answer = scan_ai._Answer() + handler = scan_ai._submit_server(source, answer).tools[0].handler + return _said(_call(handler, {"agents": agents})), answer + + +def test_the_models_paths_are_checked(tmp_path): + source = _repo(tmp_path) + _write(source / "node_modules/pkg/agent.js", "x\n") + _write(source / "build/agent.py", "x\n") + _write(source / "services/other/run.py", "x\n") + evil = _write(tmp_path / "outside/evil.py", "x\n") + os.symlink(evil, source / "services/support/evil.py") + os.symlink(evil.parent, source / "linked") + + for folder, entry, kept in ( + ("", "main.py", ("", "main.py")), + ("", AGENT[1], ("", AGENT[1])), + (AGENT[0], AGENT[1], AGENT), + ("./" + AGENT[0], "./" + AGENT[1], AGENT), + (f"{source}/{AGENT[0]}", f"{source}/{AGENT[1]}", AGENT), + (str(source), f"{source}/main.py", ("", "main.py")), + ): + said, answer = _submit(source, [{"folder": folder, "entry": entry}]) + assert said.startswith("Recorded 1; 0 refused."), (folder, entry, said) + assert answer.pairs == (kept,) + + for agent in ( + {"folder": "services/other", "entry": AGENT[1]}, + {"folder": "services/sup", "entry": AGENT[1]}, + {"folder": "", "entry": "../outside/evil.py"}, + {"folder": "", "entry": str(evil)}, + {"folder": AGENT[0], "entry": "services/support/evil.py"}, + {"folder": "linked", "entry": "linked/evil.py"}, + {"folder": "", "entry": AGENT[0]}, + {"folder": "", "entry": "missing.py"}, + {"folder": "", "entry": "node_modules/pkg/agent.js"}, + {"folder": "", "entry": "build/agent.py"}, + {"folder": "", "entry": "main.py\n"}, + {"folder": "", "entry": ""}, + {"folder": "../", "entry": "main.py"}, + {"folder": 1, "entry": "main.py"}, + "main.py", + ): + said, answer = _submit(source, [agent]) + assert said.startswith("Recorded 0; 1 refused."), (agent, said) + assert answer.pairs == () + assert answer.called is True + + said, answer = _submit(source, [{"folder": "", "entry": "main.py"}] * 2) + assert said.startswith("Recorded 1; 0 refused.") + assert answer.pairs == (("", "main.py"),) + + said, answer = _submit(source, [{"folder": "", "entry": "main.py"}] * 51) + assert answer.pairs == () + assert answer.called is False + + +def test_the_first_kept_submit_is_final_and_the_tools_stop(tmp_path, monkeypatch, quick): + backend = _Backend( + ("call", SUBMIT, {"agents": [{"folder": "", "entry": "missing.py"}]}), + KEPT, + ("call", SUBMIT, {"agents": [{"folder": "", "entry": "main.py"}]}), + ("call", READ, {"path": "main.py"}), + ("call", GLOB, {"pattern": "*"}), + ("call", GREP, {"text": "print"}), + FINISHED, + ) + + outcome = _scan(monkeypatch, _repo(tmp_path), backend) + + said = [_said(result) for result in backend.results] + assert said[0].startswith("Recorded 0; 1 refused.") + assert said[1] == "Recorded 1; 0 refused. Your work is done: end your turn now." + assert said[2] == "Already submitted." + assert said[3:] == ["Already submitted: end your turn."] * 3 + assert outcome.kind == "pairs" + assert outcome.pairs == (AGENT,) + assert outcome.refused == 0 + + +def test_a_refused_pair_in_the_kept_submit_is_counted(tmp_path, monkeypatch, quick): + agents = [ + {"folder": AGENT[0], "entry": AGENT[1]}, + {"folder": "", "entry": "missing.py"}, + {"folder": "services/other", "entry": AGENT[1]}, + ] + backend = _Backend(("call", SUBMIT, {"agents": agents}), FINISHED) + + outcome = _scan(monkeypatch, _repo(tmp_path), backend) + + assert _said(backend.results[0]).startswith("Recorded 1; 2 refused.") + assert outcome.kind == "pairs" + assert outcome.pairs == (AGENT,) + assert outcome.refused == 2 + + +def test_an_entry_reached_through_a_link_out_of_the_source_is_refused(tmp_path): + source = _repo(tmp_path) + _write(tmp_path / "outside/other.py", "x\n") + # A link in the middle of the path: the entry itself is a regular file, and its folder is + # inside the source. + os.symlink(tmp_path / "outside", source / "services/support/ext") + + for folder in (AGENT[0], ""): + said, answer = _submit(source, [{"folder": folder, "entry": f"{AGENT[0]}/ext/other.py"}]) + assert said.startswith("Recorded 0; 1 refused."), (folder, said) + assert answer.pairs == () + + +def test_the_tools_work_when_finished_is_none(tmp_path): + source = _repo(tmp_path) + tools = { + spec.name: spec.handler + for spec in scan_ai.rooted_file_tools(source, finished=None).tools + } + + result = _call(tools["read"], {"path": "main.py"}) + + assert not result.get("is_error"), _said(result) + assert "hello" in _said(result) + assert not _call(tools["glob"], {"pattern": "*.py"}).get("is_error") + + +def test_two_submits_at_once_keep_exactly_one_set(tmp_path, monkeypatch): + source = _repo(tmp_path) + + async def checked_after_a_yield(work, *args): + # A thread's answer comes back on a later turn of the loop; yielding once here makes + # the two calls interleave the same way on every run. + await asyncio.sleep(0) + return work(*args) + + monkeypatch.setattr(scan_ai, "_in_thread", checked_after_a_yield) + answer = scan_ai._Answer() + handler = scan_ai._submit_server(source, answer).tools[0].handler + first = [{"folder": AGENT[0], "entry": AGENT[1]}] + second = [{"folder": "", "entry": "main.py"}] + + async def both() -> list[dict[str, Any]]: + return list(await asyncio.gather(handler({"agents": first}), handler({"agents": second}))) + + said = [_said(result) for result in asyncio.run(asyncio.wait_for(both(), TEST_BOUND))] + + recorded = "Recorded 1; 0 refused. Your work is done: end your turn now." + assert sorted(said) == sorted([recorded, scan_ai.SUBMIT_AFTER_SUBMIT]) + assert answer.pairs == ((AGENT,), (("", "main.py"),))[said.index(recorded)] + + +# -- a path the platform can't carry (the vercel/ai-chatbot run, 6 Nov) ----------------------- + +ROUTE = "app/(chat)/api/chat/route.ts" + + +def _route_repo(tmp_path: Path) -> Path: + source = _repo(tmp_path) + _write(source / ROUTE, "export async function POST() {}\n") + return source + + +def test_a_real_file_the_path_rule_refuses_is_counted_once_and_named( + tmp_path, monkeypatch, quick +): + backend = _Backend( + ("call", SUBMIT, {"agents": [{"folder": "", "entry": ROUTE}]}), + ("call", SUBMIT, {"agents": [{"folder": "app", "entry": "./" + ROUTE}]}), + FINISHED, + ) + + outcome = _scan(monkeypatch, _route_repo(tmp_path), backend) + + said = [_said(result) for result in backend.results] + for line in said: + assert line.startswith("Recorded 0; 1 refused. 1 name a file whose path the platform") + assert "don't submit them again" in line + assert outcome.kind == "empty" + assert outcome.uncarried == 1 + + +@pytest.mark.parametrize( + "entry", + ["app/(chat)/missing.ts", "../outside/(x).ts", "/etc/(x).ts", "node_modules/(x)/a.ts"], +) +def test_a_missing_or_outside_file_is_not_counted_as_uncarried( + tmp_path, monkeypatch, quick, entry +): + source = _route_repo(tmp_path) + _write(source / "node_modules/(x)/a.ts", "x\n") + backend = _Backend(("call", SUBMIT, {"agents": [{"folder": "", "entry": entry}]}), FINISHED) + + outcome = _scan(monkeypatch, source, backend) + + assert "can't carry" not in _said(backend.results[0]) + assert outcome.kind == "empty" + assert outcome.uncarried == 0 + + +def test_an_uncarried_file_before_a_kept_submit_is_still_counted(tmp_path, monkeypatch, quick): + backend = _Backend( + ("call", SUBMIT, {"agents": [{"folder": "", "entry": ROUTE}]}), + KEPT, + FINISHED, + ) + + outcome = _scan(monkeypatch, _route_repo(tmp_path), backend) + + assert outcome.kind == "pairs" + assert outcome.pairs == (AGENT,) + assert outcome.refused == 0 + assert outcome.uncarried == 1 + + +def test_glob_reads_a_pattern_as_a_shell_would(tmp_path): + source = _route_repo(tmp_path) + _write(source / "package.json", "{}\n") + _write(source / "web/package.json", "{}\n") + _write(source / "web/src/page.tsx", "x\n") + tools = _tools(source) + + def globbed(pattern: str, **extra: str) -> set[str]: + return _listed(_said(_call(tools["glob"], {"pattern": pattern, **extra}))) + + assert globbed("**/package.json") == {"package.json", "web/package.json"} + assert globbed("**/*.{ts,tsx}") == {ROUTE, "web/src/page.tsx"} + assert globbed("web/**/page.tsx") == {"web/src/page.tsx"} + assert globbed("*.{py,json}") >= {"main.py", "package.json", AGENT[1]} + assert _said(_call(tools["glob"], {"pattern": "{"})) == "No file matches." + assert len(scan_ai._pattern_forms("{a,b}" * 20)) <= scan_ai._MAX_FORMS + found = _said(_call(tools["grep"], {"text": "POST", "glob": "**/*.{ts,tsx}"})) + assert _listed(found) == {ROUTE} diff --git a/tests/harness/test_scan_document.py b/tests/harness/test_scan_document.py new file mode 100644 index 000000000..0c8efe645 --- /dev/null +++ b/tests/harness/test_scan_document.py @@ -0,0 +1,605 @@ +from __future__ import annotations + +import json +import logging +import os +import re +import stat +from dataclasses import dataclass +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from fi.alk.harness import scan_document as sd +from fi.alk.harness.scan_document import ( + Row, + ScanResultInvalid, + choose_rows, + finish, + root_name, + wire_problems, + write_agents_json, +) + +FIXTURE = Path(__file__).parent / "fixtures" / "agent_scan_e07d56fcd.json" +SHA = "0123456789abcdef0123456789abcdef01234567" +PROBLEM_NAME = re.compile(r"[a-z0-9_.\[\]]+") + + +@dataclass(frozen=True) +class _Candidate: + """The walk's candidate, with the fields the document reads.""" + + entry: str + subfolder: str = "" + sign: str = "framework" + side: bool = False + builds: tuple[str, ...] = ("langgraph",) + build_may_stop: str = "" + partial: bool = False + env_names: tuple[str, ...] = ("OPENAI_API_KEY",) + tools: int = 2 + prompt_files: int = 1 + evidence: tuple[tuple[str, str, int], ...] = () + + +def _framework(name: str = "langgraph", label: str | None = None) -> dict[str, Any]: + return {"name": name, "version": None, "label": label} + + +def _row( + entry: str, + subfolder: str = "", + *, + framework: dict[str, Any] | None = None, + finished: bool = True, + found_by_ai: bool = False, + **candidate: Any, +) -> Row: + return Row( + candidate=_Candidate(entry=entry, subfolder=subfolder, **candidate), + framework=framework if framework is not None else _framework(), + finished=finished, + found_by_ai=found_by_ai, + ) + + +def _finish( + rows: list[Row], + *, + root: str = "shop", + commit_sha: str | None = SHA, + truncated: bool = False, + error: tuple[str, str] | None = None, +) -> tuple[dict[str, Any], bytes, bool]: + body, dropped = finish( + rows=rows, root_name=root, commit_sha=commit_sha, truncated=truncated, error=error + ) + document = json.loads(body) + assert wire_problems(document) == [] + return document, body, dropped + + +def _confidence(*rows: Row) -> list[str]: + document, _, _ = _finish(list(rows)) + return [agent["confidence"] for agent in document["agents"]] + + +def test_found_lists_each_agent_in_his_format() -> None: + graph = _row( + "services/support/graph.py", + "services/support", + env_names=("OPENAI_API_KEY", "SUPPORT_DB_URL"), + tools=3, + evidence=(("builds langgraph", "services/support/graph.py", 12),), + ) + root = _row( + "main.py", + framework=_framework("other", "llama_index"), + builds=("llama_index",), + tools=0, + prompt_files=0, + ) + + document, _, dropped = _finish([root, graph]) + + assert dropped is False + assert list(document) == [ + "schema_version", + "status", + "commit_sha", + "agents", + "truncated", + "error", + ] + assert document == { + "schema_version": "futureagi.agent-scan.v1", + "status": "found", + "commit_sha": SHA, + "agents": [ + { + "name": "support", + "subfolder": "services/support", + "entry": "services/support/graph.py", + "framework": {"name": "langgraph", "version": None, "label": None}, + "confidence": "high", + "evidence": ["Builds with LangGraph in services/support/graph.py, line 12."], + "env_names": ["OPENAI_API_KEY", "SUPPORT_DB_URL"], + "estimates": {"tools": 3, "prompt_files": 1}, + }, + { + "name": "shop", + "subfolder": "", + "entry": "main.py", + "framework": {"name": "other", "version": None, "label": "llama_index"}, + "confidence": "medium", + "evidence": [], + "env_names": ["OPENAI_API_KEY"], + "estimates": {"tools": 0, "prompt_files": 0}, + }, + ], + "truncated": False, + "error": None, + } + assert [list(agent) for agent in document["agents"]] == [list(sd._AGENT_KEYS)] * 2 + assert sd._AGENT_KEYS == ( + "name", + "subfolder", + "entry", + "framework", + "confidence", + "evidence", + "env_names", + "estimates", + ) + + none, _, _ = _finish([]) + assert (none["status"], none["agents"], none["error"]) == ("none", [], None) + error, _, _ = _finish([], error=("scan_timed_out", "The scan ran out of time.")) + assert error["status"] == "error" + assert error["error"] == {"code": "scan_timed_out", "message": "The scan ran out of time."} + assert error["truncated"] is False + + +def test_confidence_rules() -> None: + assert _confidence(_row("a.py", "a")) == ["high"] + assert _confidence(_row("a.py", "a", builds=("langchain",))) == ["medium"] + assert _confidence(_row("a.py", "a", framework=_framework("other", "dspy"))) == ["medium"] + assert _confidence(_row("a.py", "a", framework=_framework("unknown"))) == ["low"] + assert _confidence(_row("a.py", "a", framework=_framework("none"), builds=())) == ["low"] + assert _confidence(_row("a.py", "a", found_by_ai=True)) == ["low"] + assert _confidence(_row("a.py", "a", build_may_stop="graphs")) == ["low"] + assert _confidence(_row("a.py", "a", build_may_stop="agent_py")) == ["low"] + assert _confidence(_row("a.py", "a", partial=True)) == ["low"] + assert _confidence(_row("a.py", "a", finished=False)) == ["low"] + # Two agents in one folder both build cleanly, but a build of that folder can't pick one. + assert _confidence(_row("a/one.py", "a"), _row("a/two.py", "a")) == ["low", "low"] + assert _confidence(_row("a/one.py", "a"), _row("b/two.py", "b")) == ["high", "high"] + + document, _, _ = _finish( + [ + _row("tests/t.py", "tests", side=True), + _row("c/low.py", "c", framework=_framework("unknown")), + _row("b/medium.py", "b", builds=()), + _row("z/high.py", "z"), + _row("a/high.py", "a"), + ] + ) + assert [agent["entry"] for agent in document["agents"]] == [ + "a/high.py", + "z/high.py", + "b/medium.py", + "c/low.py", + "tests/t.py", + ] + + +def test_sharing_a_folder_is_decided_over_the_rows_written() -> None: + shares = sd._TEMPLATES["shares_folder"] + + # The neighbour breaks the platform's path rule, so it isn't written and nothing is shared. + document, _, dropped = _finish([_row("svc/main.py", "svc"), _row("svc/../x.py", "svc")]) + assert dropped is True + assert [agent["entry"] for agent in document["agents"]] == ["svc/main.py"] + assert document["agents"][0]["confidence"] == "high" + assert shares not in document["agents"][0]["evidence"] + + # The neighbour is the fifty-first, cut by the cap. + rows = [_row(f"f{i:02}/main.py", f"f{i:02}") for i in range(50)] + rows.append(_row("f00/tests/test_main.py", "f00", side=True)) + document, _, dropped = _finish(rows) + assert dropped is True + assert len(document["agents"]) == 50 + assert {agent["confidence"] for agent in document["agents"]} == {"high"} + + # The neighbour repeats an entry already listed. + document, _, _ = _finish( + [_row("svc/main.py", "svc"), _row("svc/main.py", "svc", found_by_ai=True)] + ) + assert [(agent["entry"], agent["confidence"]) for agent in document["agents"]] == [ + ("svc/main.py", "high") + ] + + document, _, _ = _finish([_row("svc/main.py", "svc"), _row("svc/other.py", "svc")]) + assert all(shares in agent["evidence"] for agent in document["agents"]) + + +def test_an_uppercase_commit_is_lowercased() -> None: + assert _finish([_row("a.py")], commit_sha=SHA.upper())[0]["commit_sha"] == SHA + assert _finish([_row("a.py")], commit_sha="0123456789ABCdef" * 2 + "01234567")[0][ + "commit_sha" + ] == SHA + assert _finish([_row("a.py")], commit_sha=None)[0]["commit_sha"] is None + assert _finish([_row("a.py")], commit_sha=SHA[:39])[0]["commit_sha"] is None + assert _finish([_row("a.py")], commit_sha="g" * 40)[0]["commit_sha"] is None + assert _finish([_row("a.py")], commit_sha=SHA + "\n")[0]["commit_sha"] is None + + +def test_rows_and_an_error_together_keep_the_rows() -> None: + error = ("scan_timed_out", "The scan ran out of time.") + + document, _, dropped = _finish([_row("a/main.py", "a")], error=error) + assert document["status"] == "found" + assert document["error"] is None + assert document["truncated"] is True + assert [agent["entry"] for agent in document["agents"]] == ["a/main.py"] + assert dropped is False + + # Every row given breaks a rule: nothing is left to list, and the error was already dropped. + document, _, dropped = _finish([_row("a/../main.py", "a")], error=error) + assert (document["status"], document["agents"], document["error"]) == ("none", [], None) + assert document["truncated"] is True + assert dropped is True + + +def test_names_evidence_and_paths_always_fit_his_rules() -> None: + def job(agent_name: Any = None, repository: Any = None) -> Any: + metadata = {} if agent_name is None else {"agent_name": agent_name} + return SimpleNamespace(metadata=metadata, source=SimpleNamespace(repository=repository)) + + assert root_name(job(" Support\u2003Bot\x00 ", "org/shop.git")) == "Support Bot" + assert root_name(job(3, "org/shop.git")) == "shop" + assert root_name(job("\u200b\u3000", "https://github.com/org/shop.git/")) == "shop" + assert root_name(job(None, "shop")) == "shop" + assert root_name(job(None, ".git")) == "agent" + assert root_name(job()) == "agent" + + document, _, _ = _finish( + [ + _row("a/svc/main.py", "a/svc"), + _row("b/svc/app.py", "b/svc"), + _row("c/api/main.py", "c/api"), + _row("d/api/main.py", "d/api"), + _row("main.py"), + _row("server.py"), + ], + root="shop", + ) + assert sorted(agent["name"] for agent in document["agents"]) == [ + "api (c/api/main.py)", + "api (d/api/main.py)", + "shop (main.py)", + "shop (server.py)", + "svc (app.py)", + "svc (main.py)", + ] + assert _finish([_row("main.py")], root="x" * 300)[0]["agents"][0]["name"] == "x" * 255 + assert _finish([_row("main.py")], root="\u200b\x00")[0]["agents"][0]["name"] == "agent" + + long_path = "deep/" * 90 + "graph.py" + facts = tuple(("builds langgraph", f"f{i}/graph.py", i + 1) for i in range(7)) + row = _row( + long_path, + "deep", + evidence=(("builds langgraph", long_path, 4), *facts), + env_names=tuple(f"NAME_{i}" for i in range(60)), + build_may_stop="graphs", + partial=True, + finished=False, + ) + agent = _finish([row])[0]["agents"][0] + assert len(agent["evidence"]) == 5 + assert all(0 < len(sentence) <= 200 for sentence in agent["evidence"]) + assert agent["evidence"][0].startswith("Builds with LangGraph in …/") + assert agent["evidence"][0].endswith("deep/graph.py, line 4.") + assert agent["evidence"][1:4] == [ + sd._TEMPLATES["graphs_may_stop"], + sd._TEMPLATES["not_finished"], + sd._TEMPLATES["partial"], + ] + assert agent["env_names"] == [f"NAME_{i}" for i in range(50)] + + assert sd._LONGEST_TEMPLATE == max(len(template) for template in sd._TEMPLATES.values()) + assert sd._LONGEST_TEMPLATE + sd._PATH_ROOM <= sd.EVIDENCE_MAX + assert sd._PATH_ROOM == 120 + + aside = (("openai_agents_set_aside", "", 0),) + assert sd._TEMPLATES["openai_agents_set_aside"] in _finish( + [_row("a/main.py", "a", framework=_framework("unknown"), evidence=aside)] + )[0]["agents"][0]["evidence"] + assert _finish( + [_row("a/main.py", "a", framework=_framework("openai_agents"), evidence=aside)] + )[0]["agents"][0]["evidence"] == [] + + +def test_clean_name_collapses_every_kind_of_space() -> None: + assert sd._clean_name("a\u2003\u3000b") == "a b" + assert sd._clean_name("\u3000a \t\n b\u2028c\u00a0d\u205f ") == "a b c d" + assert sd._clean_name("a\u200bb\x00c\x1bd\ue000e\ud800f\u202eg\ufeffh") == "abcdefgh" + assert sd._clean_name("\u3000\u2003\u200b") == "" + assert _finish([_row("main.py")], root="Shop\u3000\u2003Bot")[0]["agents"][0]["name"] == ( + "Shop Bot" + ) + + +def test_a_sentence_never_names_a_path_his_rule_refuses() -> None: + undecodable = os.fsdecode(b"agent\xff.py") + refused = [ + undecodable, + "../agent.py", + "/srv/agent.py", + "my agent.py", + "a//agent.py", + "agent.py\n", + "x" * 513, + "", + None, + ] + for kind in ("builds langgraph", "graph", "agent_py", "client openai", "project"): + for path in refused: + sentence = sd._sentence(kind, path, 3) + assert sentence is not None + assert sd._UNNAMED_PATH in sentence + if path: + assert path not in sentence + assert "\udcff" not in sentence + sentence.encode("utf-8") + assert len(sentence) <= 200 + + long_path = "d/" * 200 + "agent.py" + sentence = sd._sentence("builds langgraph", long_path, 3) + assert "…/" in sentence and sentence.endswith("/d/agent.py, line 3.") + assert len(sentence) <= 200 + + one_long_part = "a" * 300 + ".py" + sentence = sd._sentence("agent_py", one_long_part, 0) + assert "…" in sentence and len(sentence) <= 200 + + assert sd._sentence("client open ai!", "app.py", 1) == ( + "Creates a client for a model provider in app.py, line 1." + ) + assert sd._sentence("client openai", "app.py", 1) == ( + "Creates a client for openai in app.py, line 1." + ) + assert sd._sentence("builds \x00", "app.py", 1) == ( + "Builds with an agent framework in app.py, line 1." + ) + assert sd._sentence("no_such_fact", "app.py", 1) is None + + +def test_the_file_is_compact_utf8_and_never_fails_on_broken_characters() -> None: + undecodable = os.fsdecode(b"agent\xff.py") + rows = [ + _row("main.py", evidence=(("builds langgraph", undecodable, 2),)), + _row("x" + undecodable, "x" + undecodable), + ] + + document, body, dropped = _finish(rows, root="Café\ud800 Bot") + + assert body == json.dumps(document, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + body.decode("utf-8") + assert "Café Bot".encode("utf-8") in body + assert b"\\u00e9" not in body + assert b"\n" not in body + assert dropped is True + assert [agent["entry"] for agent in document["agents"]] == ["main.py"] + assert document["agents"][0]["evidence"] == [ + "Builds with LangGraph in a file in this folder, line 2." + ] + + +def test_finish_never_writes_more_than_fifty() -> None: + rows = [_row(f"f{i:02}/main.py", f"f{i:02}") for i in range(60)] + + document, _, dropped = _finish(rows) + + assert len(document["agents"]) == 50 + assert document["truncated"] is True + assert dropped is True + assert [agent["entry"] for agent in document["agents"]] == [ + f"f{i:02}/main.py" for i in range(50) + ] + + fifty, _, dropped = _finish(rows[:50]) + assert (len(fifty["agents"]), fifty["truncated"], dropped) == (50, False, False) + + first = _row("a/main.py", "a") + again = _row("a/main.py", "b", found_by_ai=True) + chosen, left_out = choose_rows([_row("z/main.py", "z"), first, again], limit=50) + assert [row.candidate.entry for row in chosen] == ["a/main.py", "z/main.py"] + assert chosen[0] is first + assert left_out is True + chosen, left_out = choose_rows(rows, limit=3) + assert (len(chosen), left_out) == (3, True) + assert choose_rows(rows[:3]) == (sorted(rows[:3], key=lambda row: row.candidate.entry), False) + + +def test_the_models_rows_are_chosen_first_only_when_asked() -> None: + graph = _row("a/graph.py", "a") + helpers = [_row(f"{name}/main.py", name, sign="client", builds=()) for name in ("b", "c")] + found = _row("z/agent.ts", "z", found_by_ai=True, builds=()) + in_tests = _row("tests/agent.ts", "tests", side=True, found_by_ai=True, builds=()) + rows = [graph, *helpers, found, in_tests] + + def entries(chosen: list[Row]) -> list[str]: + return [row.candidate.entry for row in chosen] + + plain, plain_left_out = choose_rows(rows, limit=3) + first, first_left_out = choose_rows(rows, limit=3, models_first=True) + many = [_row(f"h{i:02}/main.py", f"h{i:02}", sign="client", builds=()) for i in range(49)] + body, dropped = finish( + rows=[graph, *many, found], + root_name="shop", + commit_sha=SHA, + truncated=False, + error=None, + models_first=True, + ) + written = [agent["entry"] for agent in json.loads(body)["agents"]] + + assert _confidence(graph, *helpers, found) == ["high", "medium", "medium", "low"] + assert (entries(plain), plain_left_out) == (["a/graph.py", "b/main.py", "c/main.py"], True) + assert (entries(first), first_left_out) == (["a/graph.py", "z/agent.ts", "b/main.py"], True) + # Which rows are kept changes; the order they are written in does not. + assert dropped is True + assert written == ["a/graph.py", *(f"h{i:02}/main.py" for i in range(48)), "z/agent.ts"] + + +def test_a_row_his_rules_reject_is_dropped_with_truncated(caplog: pytest.LogCaptureFixture) -> None: + good = _row("good/main.py", "good") + bad = [ + _row("bad/../escape.py", "bad"), + _row("label/main.py", "label", framework={"name": "langgraph", "version": None}), + _row("env/main.py", "env", env_names=("BAD-NAME-sk-fake-123",)), + _row("tools/main.py", "tools", tools=-1), + _row("word/main.py", "word", framework=_framework("autogen")), + ] + + with caplog.at_level(logging.WARNING, logger="fi.alk.harness.scan_document"): + document, _, dropped = _finish([good, *bad]) + + assert [agent["entry"] for agent in document["agents"]] == ["good/main.py"] + assert document["status"] == "found" + assert document["truncated"] is True + assert dropped is True + left_out = [record.getMessage() for record in caplog.records] + assert len(left_out) == len(bad) + assert all(message.startswith("agent scan: row left out, problem=") for message in left_out) + assert "sk-fake-123" not in caplog.text + assert "escape" not in caplog.text + + document, _, dropped = _finish(bad) + assert (document["status"], document["agents"], document["truncated"]) == ("none", [], True) + assert dropped is True + + +def test_a_bad_top_level_raises(monkeypatch: pytest.MonkeyPatch) -> None: + def problem(**changes: Any) -> str: + arguments: dict[str, Any] = { + "rows": [], + "root_name": "shop", + "commit_sha": SHA, + "truncated": False, + "error": None, + } + arguments.update(changes) + with pytest.raises(ScanResultInvalid) as raised: + finish(**arguments) + assert raised.value.problem in str(raised.value) + return raised.value.problem + + assert problem(error=("Scan-Failed", "The scan failed.")) == "error.code.pattern" + assert problem(error=("scan_failed", " ")) == "error.message.blank" + assert problem(error=("scan_failed", "m" * 2001)) == "error.message.too_long" + assert problem(error=("scan_failed", "sk-fake-123\x00")) == "error.message.null_byte" + assert "sk-fake-123" not in problem(error=("sk-fake-123", "The scan failed.")) + + monkeypatch.setattr(sd, "MAX_BYTES", 100) + assert problem(rows=[_row("main.py")]) == "document.too_large" + + assert wire_problems([]) == ["not_an_object"] + assert wire_problems({"schema_version": "futureagi.agent-scan.v1"}) == [ + "status.missing", + "commit_sha.missing", + "agents.missing", + "truncated.missing", + "error.missing", + ] + + +def test_the_file_matches_his_serializer() -> None: + fixture = json.loads(FIXTURE.read_text(encoding="utf-8")) + assert fixture["serializer_commit"] == "e07d56fcd" + cases = fixture["cases"] + assert len(cases) > 100 + assert {case["accepted"] for case in cases} == {True, False} + # Lone surrogates and null bytes reach the reader only as escapes in the text. + assert any("\\ud800" in case["document_text"] for case in cases) + assert any("\\u0000" in case["document_text"] for case in cases) + + disagreements = [] + for case in cases: + problems = wire_problems(json.loads(case["document_text"])) + if (problems == []) != case["accepted"]: + disagreements.append((case["name"], case["accepted"], problems)) + assert all(PROBLEM_NAME.fullmatch(problem) for problem in problems), case["name"] + assert disagreements == [] + + +def test_a_failed_write_leaves_no_agents_json( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _, body, _ = _finish([_row("main.py")]) + + written = write_agents_json(tmp_path / "ok", body) + assert written == tmp_path / "ok" / "agents.json" + assert written.read_bytes() == body + assert stat.S_IMODE(written.stat().st_mode) == 0o644 + assert [path.name for path in (tmp_path / "ok").iterdir()] == ["agents.json"] + + def refuse(*args: Any, **kwargs: Any) -> None: + raise OSError("disk full") + + for name in ("replace", "fsync"): + output = tmp_path / name + output.mkdir() + with monkeypatch.context() as patch: + patch.setattr(sd.os, name, refuse) + with pytest.raises(OSError): + write_agents_json(output, body) + assert list(output.iterdir()) == [] + + earlier = tmp_path / "earlier" + earlier.mkdir() + (earlier / "agents.json").write_bytes(b"{}") + with monkeypatch.context() as patch: + patch.setattr(sd.os, "replace", refuse) + with pytest.raises(OSError): + write_agents_json(earlier, body) + assert [path.name for path in earlier.iterdir()] == ["agents.json"] + assert (earlier / "agents.json").read_bytes() == b"{}" + + +def test_a_disambiguating_suffix_survives_the_255_cut() -> None: + document, _, _ = _finish([_row("a/main.py"), _row("b/main.py")], root="x" * 300) + + names = sorted(agent["name"] for agent in document["agents"]) + assert names == [ + "x" * (255 - len(" (a/main.py)")) + " (a/main.py)", + "x" * (255 - len(" (b/main.py)")) + " (b/main.py)", + ] + + +def test_a_chat_model_row_is_low_with_its_own_sentence() -> None: + evidence = (("model langchain", "app/bot.py", 4),) + + agent = _finish( + [ + _row( + "app/bot.py", + "app", + sign="model", + builds=(), + evidence=evidence, + framework=_framework("langchain"), + ) + ] + )[0]["agents"][0] + + assert agent["confidence"] == "low" + assert agent["evidence"] == ["Calls a chat model through LangChain in app/bot.py, line 4."] + assert sd._sentence("model no such!", "app.py", 1) == ( + "Calls a chat model through an agent framework in app.py, line 1." + ) diff --git a/tests/harness/test_scan_folders.py b/tests/harness/test_scan_folders.py new file mode 100644 index 000000000..91a4a2f32 --- /dev/null +++ b/tests/harness/test_scan_folders.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +import multiprocessing.process +import os +import signal +import time +from pathlib import Path +from typing import Any + +import pytest + +from fi.alk.harness import scan_folders +from fi.alk.harness.framework_detection import detect_framework +from fi.alk.harness.scan_folders import ( + ClockRanOut, + FolderChecker, + FolderCheckerBroken, + FolderWord, +) +from fi.alk.harness.source_folder import agent_folder + +CREW = "from crewai import Crew\nCrew()\n" +AUTOGEN = "from autogen_agentchat.agents import AssistantAgent\nAssistantAgent()\n" +OPENAI_AGENT = "from agents import Agent\nAgent()\n" +PLAIN = "print('hello')\n" +SITE = ".venv/lib/python3.12/site-packages" +# The first check in a session also starts the fork server, which imports the kit. +FIRST_CHECK_SECONDS = 120.0 +# Killing and reaping a stopped child, on a loaded machine. +KILL_MARGIN_SECONDS = 4.0 +UNKNOWN = {"name": "unknown", "version": None, "label": None} +FOLDERS = { + "": None, + "crew": {"name": "crewai", "version": None, "label": None}, + "autogen": {"name": "other", "version": None, "label": "autogen"}, + "oa": {"name": "openai_agents", "version": None, "label": None}, + "plain": {"name": "none", "version": None, "label": None}, +} + + +def _write(folder: Path, files: dict[str, str]) -> None: + for relative, content in files.items(): + path = folder / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def _repo(tmp_path: Path) -> Path: + source = tmp_path / "repo" + _write( + source, + { + "crew/main.py": CREW, + "autogen/main.py": AUTOGEN, + "oa/main.py": OPENAI_AGENT, + "plain/main.py": PLAIN, + # Installed packages without openai-agents: a check that read them would drop + # the `agents` import, so the word shows which package folder the child used. + f"{SITE}/requests-2.0.dist-info/METADATA": "Version: 2.0\n", + f"oa/{SITE}/requests-2.0.dist-info/METADATA": "Version: 2.0\n", + }, + ) + return source + + +def _later(seconds: float) -> float: + return time.monotonic() + seconds + + +def _gone(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + return False + + +def _kill_if_alive(pid: int | None) -> None: + if pid is not None and not _gone(pid): + os.kill(pid, signal.SIGKILL) + + +def _exits_three(conn: Any) -> None: + os._exit(3) + + +def _refuses_to_start(self: Any) -> None: + raise OSError("no processes left") + + +def test_each_folder_gets_the_builds_framework_word(tmp_path: Path) -> None: + source = _repo(tmp_path) + words: dict[str, FolderWord] = {} + pids = set() + with FolderChecker() as checker: + for subfolder in FOLDERS: + words[subfolder] = checker.check( + source, subfolder, deadline=_later(FIRST_CHECK_SECONDS) + ) + pids.add(checker.pid) + + for subfolder, expected in FOLDERS.items(): + answer = detect_framework( + agent_folder(source, subfolder) if subfolder else source, + repo_root=source, + package_dir=None, + ) + assert words[subfolder] == FolderWord( + {"name": answer["name"], "version": None, "label": answer.get("label")}, + finished=True, + ) + assert set(words[subfolder].framework) == {"name", "version", "label"} + if expected is not None: + assert words[subfolder].framework == expected + assert len(pids) == 1 + assert None not in pids and os.getpid() not in pids + + +def test_the_clock_ends_a_stopped_child(tmp_path: Path) -> None: + source = _repo(tmp_path) + stopped: int | None = None + try: + with FolderChecker() as checker: + checker.check(source, "crew", deadline=_later(FIRST_CHECK_SECONDS)) + stopped = checker.pid + assert stopped is not None + os.kill(stopped, signal.SIGSTOP) + + started = time.monotonic() + with pytest.raises(ClockRanOut): + checker.check(source, "plain", deadline=started + 1.0) + + assert time.monotonic() - started < 1.0 + KILL_MARGIN_SECONDS + assert checker.pid is None + assert _gone(stopped) + + with pytest.raises(ClockRanOut): + checker.check(source, "autogen", deadline=time.monotonic() - 1.0) + assert checker.pid is None + + word = checker.check(source, "plain", deadline=_later(FIRST_CHECK_SECONDS)) + assert word == FolderWord(FOLDERS["plain"], finished=True) + assert checker.pid not in (None, stopped) + finally: + _kill_if_alive(stopped) + + +def test_the_child_is_restarted_after_a_kill(tmp_path: Path) -> None: + source = _repo(tmp_path) + killed: int | None = None + try: + with FolderChecker() as checker: + first = checker.check(source, "crew", deadline=_later(FIRST_CHECK_SECONDS)) + assert first == FolderWord(FOLDERS["crew"], finished=True) + killed = checker.pid + assert killed is not None + os.kill(killed, signal.SIGKILL) + + word = checker.check(source, "autogen", deadline=_later(FIRST_CHECK_SECONDS)) + + assert word == FolderWord(UNKNOWN, finished=False) + assert _gone(killed) + + again = checker.check(source, "plain", deadline=_later(FIRST_CHECK_SECONDS)) + assert again == FolderWord(FOLDERS["plain"], finished=True) + assert checker.pid not in (None, killed) + finally: + _kill_if_alive(killed) + + +def test_a_child_that_cannot_start_is_broken( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = _repo(tmp_path) + + # The target is looked up when the child starts, so the child runs this one and exits 3. + monkeypatch.setattr(scan_folders, "_serve", _exits_three) + with FolderChecker() as checker: + with pytest.raises(FolderCheckerBroken, match="exit code 3"): + checker.check(source, "crew", deadline=_later(FIRST_CHECK_SECONDS)) + assert checker.pid is None + monkeypatch.undo() + + monkeypatch.setattr(multiprocessing.process.BaseProcess, "start", _refuses_to_start) + with FolderChecker() as checker: + with pytest.raises(FolderCheckerBroken, match="could not start"): + checker.check(source, "crew", deadline=_later(FIRST_CHECK_SECONDS)) + assert checker.pid is None + + +def test_exit_never_raises(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + source = _repo(tmp_path) + + assert FolderChecker().__exit__(None, None, None) is None + + with FolderChecker() as checker: + checker.check(source, "crew", deadline=_later(FIRST_CHECK_SECONDS)) + ended = checker.pid + assert ended is not None + assert checker.pid is None + assert _gone(ended) + assert checker.__exit__(None, None, None) is None + + stuck = FolderChecker() + stuck.check(source, "crew", deadline=_later(FIRST_CHECK_SECONDS)) + alive = stuck.pid + assert alive is not None + real_kill = os.kill + + def refuse(pid: int, sig: int) -> None: + raise PermissionError("not allowed") + + monkeypatch.setattr(os, "kill", refuse) + try: + assert stuck.__exit__(None, None, None) is None + assert stuck.pid is None + finally: + monkeypatch.undo() + try: + real_kill(alive, signal.SIGKILL) + except ProcessLookupError: + pass diff --git a/tests/harness/test_scan_walk.py b/tests/harness/test_scan_walk.py new file mode 100644 index 000000000..243c0df72 --- /dev/null +++ b/tests/harness/test_scan_walk.py @@ -0,0 +1,1201 @@ +from __future__ import annotations + +import ast +import inspect +import io +import json +import os +import tokenize +from pathlib import Path + +import pytest + +from fi.alk.harness import framework_detection as fd +from fi.alk.harness import scan_walk +from fi.alk.harness import source_folder +from fi.alk.harness.scan_walk import ( + Candidate, + WalkResult, + build_hazard, + describe_entry, + fits_path_rule, + walk_repo, +) + +GRAPH = "from langgraph.graph import StateGraph\nStateGraph(dict)\n" +CLIENT = "from openai import OpenAI\nOpenAI()\n" +FOREVER = float("inf") +ALLOWANCE = 1_000_000_000 + + +def _write(folder: Path, files: dict[str, str]) -> None: + for relative, content in files.items(): + path = folder / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def _walk(source: Path, **kwargs: object) -> WalkResult: + return walk_repo(source, deadline=FOREVER, **kwargs) # type: ignore[arg-type] + + +def _entries(result: WalkResult) -> list[str]: + return [agent.entry for agent in result.agents] + + +def _agent(result: WalkResult, entry: str) -> Candidate: + return next(agent for agent in result.agents if agent.entry == entry) + + +def _snapshot(folder: Path) -> list[tuple[str, int, int]]: + seen = [] + for directory, directories, files in os.walk(folder, followlinks=False): + for name in sorted(directories + files): + path = os.path.join(directory, name) + status = os.lstat(path) + seen.append((path, status.st_size, status.st_mtime_ns)) + return sorted(seen) + + +def _token_work(source: str) -> int: + tokens = tokenize.tokenize(io.BytesIO(source.encode()).readline) + return sum(len(token.line) for token in tokens) + + +def test_the_walk_skips_what_the_build_skips_and_never_follows_a_link( + tmp_path: Path, +) -> None: + outside = tmp_path / "outside" + _write(outside, {"agent_graph.py": GRAPH}) + source = tmp_path / "source" + ran = tmp_path / "ran" + _write( + source, + { + "app/graph.py": GRAPH + f"open({str(ran)!r}, 'w').close()\n", + "app/build/graph.py": GRAPH, + "build/graph.py": GRAPH, + "env/graph.py": GRAPH, + "node_modules/pkg/graph.py": GRAPH, + ".venv/lib/graph.py": GRAPH, + "lib/site-packages/graph.py": GRAPH, + "lib/site/pkg-1.0.dist-info/METADATA": "Name: pkg\n", + "lib/site/graph.py": GRAPH, + }, + ) + (source / "linked").symlink_to(outside, target_is_directory=True) + (source / "linked_graph.py").symlink_to(outside / "agent_graph.py") + before = _snapshot(tmp_path) + + result = _walk(source) + + assert sorted(_entries(result)) == ["app/build/graph.py", "app/graph.py"] + assert result.unread_files == 1 + assert result.stopped == "" + assert not ran.exists() + assert _snapshot(tmp_path) == before + + +def test_a_heavy_file_uses_only_its_own_allowance(tmp_path: Path) -> None: + # Every mention of `C`, in its imports and its uses, walks all of its targets. + targets = 500 + imports = "".join(f"import crewai.m{index} as C\n" for index in range(targets)) + light = imports + "C.Crew()\n" * 100 + heavy = imports + "C.Crew()\n" * 1_500 + per_mention = fd._WORK_PER_TARGET * targets + light_work = (targets + 100) * per_mention + 3 * _token_work(light) + heavy_work = (targets + 1_500) * per_mention + assert scan_walk._FILE_WORK == ALLOWANCE + assert light_work < ALLOWANCE < 2 * light_work + assert ALLOWANCE < heavy_work < fd._MAX_WORK + _write(tmp_path, {"a_light.py": light, "b_light.py": light, "c_heavy.py": heavy}) + + result = _walk(tmp_path) + + assert _entries(result) == ["a_light.py", "b_light.py"] + assert result.unread_files == 1 + assert result.stopped == "" + + +def test_the_private_detector_names_it_uses_exist() -> None: + for name in ( + "_ALWAYS_SKIPPED", + "_TOP_ONLY_SKIPS", + "_FileScan", + "_has_long_line", + "_is_side_file", + "_is_side_part", + "_OverBudget", + "_own_modules", + "_read_file", + "_scan_file", + "_visible_tokens", + ): + assert hasattr(fd, name), name + assert hasattr(source_folder, "_FOLDER_FORM") + assert source_folder._FOLDER_MAX == 512 + assert list(inspect.signature(fd._FileScan).parameters) == [ + "shadowed", + "outside_names", + "work", + ] + assert "read_uses" in inspect.signature(fd._scan_file).parameters + from fi.alk.harness import bundle_author_v2 + + assert scan_walk._BUILD_IGNORED == frozenset( + bundle_author_v2._IGNORED_ARTIFACT_PARTS + ) + + +@pytest.mark.parametrize( + "shape", + [ + b'x = """never closed\n', + b"if True:\n a = 1\n b = 2\n", + b"if True:\n\ta = 1\n b = 2\n", + b"# caf\xe9\nx = 1\n", + b"x = 1\x00\n", + ], + ids=["unclosed_string", "bad_dedent", "tab_and_spaces", "latin1_no_coding", "null"], +) +def test_a_file_the_tokenizer_rejects_is_unread_not_a_crash( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, shape: bytes +) -> None: + (tmp_path / "a_shape.py").write_bytes(GRAPH.encode() + shape) + _write(tmp_path, {"z_agent.py": GRAPH}) + + result = _walk(tmp_path) + + assert result.stopped == "" + assert "z_agent.py" in _entries(result) + assert ("a_shape.py" in _entries(result)) != (result.unread_files == 1) + + def rejects(data: bytes, work: object) -> object: + raise LookupError("a reader fault of any kind") + + monkeypatch.setattr(scan_walk, "_visible_tokens", rejects) + failing = _walk(tmp_path) + + assert failing.stopped == "" + assert failing.agents == () + assert failing.unread_files == 2 + + +def test_the_token_pass_writes_no_warnings( + tmp_path: Path, capfd: pytest.CaptureFixture[str], recwarn: pytest.WarningsRecorder +) -> None: + _write( + tmp_path, + {"agent.py": CLIENT + ("import os\nos.getenv('A\\{')\nx = f'\\{a}'\n" * 3)}, + ) + + _walk(tmp_path) + + assert not recwarn.list + assert "Warning" not in capfd.readouterr().err + + +def test_a_relative_import_joins_the_sub_agent(tmp_path: Path) -> None: + _write( + tmp_path, + { + "app/__init__.py": "", + "app/main.py": GRAPH + + "from . import planner\n" + + "def build():\n from .researcher import agent\n return agent\n", + "app/planner.py": GRAPH, + "app/researcher.py": GRAPH, + "deep/a/b/main.py": GRAPH + + "from ..helpers import util\nfrom .. import (shared)\n", + "deep/a/helpers/util.py": GRAPH, + "deep/a/shared.py": GRAPH, + "pkg/pyproject.toml": "", + "pkg/run.py": GRAPH + "import tools.search\n", + "pkg/src/tools/search.py": GRAPH, + }, + ) + + result = _walk(tmp_path) + + assert sorted(_entries(result)) == ["app/main.py", "deep/a/b/main.py", "pkg/run.py"] + + +def test_settings_tools_and_clients_are_read(tmp_path: Path) -> None: + source = tmp_path / "one" + _write( + source, + { + "bot.py": "import os\n" + + CLIENT + + "a = os.getenv('FIRST')\n" + + 'b = os.environ.get("SECOND")\n' + + "c = os.environ['THIRD']\n" + + "d = os.getenv(f'FOURTH')\n" + + "e = os.getenv('FIF' 'TH')\n" + + "f = os.getenv('SIX' + 'TH')\n" + + "g = os.getenv(NAME)\n" + + "h = os.environ[b'EIGHTH']\n" + + "def handler():\n return os.getenv( 'NINTH' )\n" + + "@tool\ndef one(): ...\n" + + "@function_tool\ndef two(): ...\n" + + "@mcp.tool()\ndef three(): ...\n" + + "@agents.function_tool(name='x')\ndef four(): ...\n" + + "@toolkit\ndef not_a_tool(): ...\n" + }, + ) + + [agent] = _walk(source).agents + + assert agent.sign == "client" + assert agent.env_names == ("FIRST", "NINTH", "SECOND", "THIRD") + assert agent.tools == 4 + assert [(kind, path) for kind, path, _ in agent.evidence] == [ + ("client openai", "bot.py") + ] + + forms = { + "openai.py": "OpenAI()", + "openai_async.py": "AsyncOpenAI()", + "openai_azure.py": "openai.AzureOpenAI()", + "anthropic.py": "Anthropic()", + "anthropic_async.py": "AsyncAnthropic()", + "genai.py": "genai.Client()", + "mistral.py": "Mistral(api_key=k)", + "cohere.py": "cohere.Client()", + "cohere_v2.py": "cohere.ClientV2()", + "groq.py": "Groq()", + "litellm.py": "litellm.completion(model=m)", + "litellm_async.py": "litellm.acompletion(model=m)", + } + clients = tmp_path / "clients" + _write(clients, {name: call + "\n" for name, call in forms.items()}) + _write(clients, {"not_a_client.py": "other.Client()\nOpenAI\n"}) + + assert sorted(_entries(_walk(clients))) == sorted(forms) + + +def test_a_getenv_default_is_never_an_env_name(tmp_path: Path) -> None: + _write( + tmp_path, + { + "bot.py": "import os\n" + + CLIENT + + "key = os.getenv('API_KEY', 'sk-fake-123')\n" + + "url = os.getenv('BASE_URL', 'FALLBACK_NAME')\n" + + "region = os.environ.get('REGION', 'DEFAULT_REGION')\n" + + "model = os.getenv('MODEL', default='DEFAULT_MODEL')\n" + }, + ) + + [agent] = _walk(tmp_path).agents + + assert agent.env_names == ("API_KEY", "BASE_URL", "MODEL", "REGION") + assert "sk-fake-123" not in repr(agent) + + +def test_env_files_with_values_are_never_opened( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write( + tmp_path, + { + ".env": "REAL_SECRET=sk-fake-123\n", + ".env.local": "LOCAL_SECRET=sk-fake-456\n", + "app/.env": "APP_SECRET=sk-fake-789\n", + ".env.example": "# a note\nEXAMPLE_NAME=sk-fake-000\n" + "export EXPORTED_NAME=1\n BAD-NAME=2\n", + "bot.py": CLIENT, + }, + ) + opened: list[str] = [] + read = scan_walk._read_file + + def spy(path: str, limit: int) -> bytes: + opened.append(os.path.basename(path)) + return read(path, limit) + + monkeypatch.setattr(scan_walk, "_read_file", spy) + + result = _walk(tmp_path) + + assert ".env.example" in opened + assert ".env" not in opened + assert ".env.local" not in opened + [agent] = result.agents + assert agent.env_names == ("EXAMPLE_NAME", "EXPORTED_NAME") + assert "sk-fake" not in repr(result) + + +@pytest.mark.parametrize( + "content", + [b"{not json", b"[" * 100_000 + b"]" * 100_000, b'{"graphs": "\xff"}'], + ids=["not_json", "deep", "not_utf8"], +) +def test_a_broken_langgraph_json_is_unread_not_a_crash( + tmp_path: Path, content: bytes +) -> None: + (tmp_path / "langgraph.json").write_bytes(content) + _write(tmp_path, {"graph.py": GRAPH}) + + result = _walk(tmp_path) + + assert result.stopped == "" + assert result.unread_files == 1 + assert _entries(result) == ["graph.py"] + + +def test_the_walk_limit_and_the_clock_stop_it( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + assert (scan_walk._MAX_FILES, scan_walk._MAX_BYTES) == (20_000, 268_435_456) + _write(tmp_path, {"a.py": GRAPH, "b.py": GRAPH, "c.py": GRAPH}) + calls: list[int] = [] + + def clock() -> float: + calls.append(1) + return 0.0 if len(calls) == 1 else 100.0 + + timed = walk_repo(tmp_path, deadline=10.0, clock=clock) + + assert timed.stopped == "clock" + assert _entries(timed) == ["a.py"] + assert len(calls) == 2 + + monkeypatch.setattr(scan_walk, "_MAX_FILES", 2) + by_count = _walk(tmp_path) + assert by_count.stopped == "limit" + assert _entries(by_count) == ["a.py", "b.py"] + + monkeypatch.setattr(scan_walk, "_MAX_FILES", 20_000) + monkeypatch.setattr(scan_walk, "_MAX_BYTES", len(GRAPH)) + by_bytes = _walk(tmp_path) + assert by_bytes.stopped == "limit" + assert _entries(by_bytes) == ["a.py"] + + +def test_a_memory_error_stops_the_walk_with_its_facts_dropped( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write(tmp_path, {"a.py": GRAPH, "b.py": GRAPH}) + walks: list[scan_walk._Walk] = [] + + class Recording(scan_walk._Walk): + def __init__(self, *args: object, **kwargs: object) -> None: + super().__init__(*args, **kwargs) # type: ignore[arg-type] + walks.append(self) + + read = scan_walk._read_file + + def runs_out(path: str, limit: int) -> bytes: + if path.endswith("b.py"): + raise MemoryError() + return read(path, limit) + + monkeypatch.setattr(scan_walk, "_Walk", Recording) + monkeypatch.setattr(scan_walk, "_read_file", runs_out) + + result = _walk(tmp_path) + + assert result.stopped == "memory" + assert result.agents == () + [walk] = walks + assert walk.facts == {} + + +def test_a_weak_candidate_does_not_swallow_a_strong_one(tmp_path: Path) -> None: + _write( + tmp_path, + { + "app/pyproject.toml": "", + "app/graph.py": GRAPH + "from llm import ask\n", + "app/llm.py": CLIENT, + "evaluation/pyproject.toml": "", + "evaluation/run.py": CLIENT + "from app.graph import graph\n", + }, + ) + + result = _walk(tmp_path) + + assert sorted(_entries(result)) == ["app/graph.py", "evaluation/run.py"] + + +def test_a_side_folder_script_does_not_swallow_the_agent(tmp_path: Path) -> None: + _write( + tmp_path, + { + "app/graph.py": GRAPH, + "evals/run.py": GRAPH + "from app.graph import graph\n", + }, + ) + + result = _walk(tmp_path) + + assert [(agent.entry, agent.side) for agent in result.agents] == [ + ("app/graph.py", False), + ("evals/run.py", True), + ] + assert result.candidate_outside_side + + +def test_an_import_cycle_is_one_agent(tmp_path: Path) -> None: + direct = tmp_path / "direct" + _write( + direct, + {"long_name.py": GRAPH + "import b\n", "b.py": GRAPH + "import long_name\n"}, + ) + through = tmp_path / "through" + _write( + through, + { + "c2.py": GRAPH + "import c1\n", + "c1.py": GRAPH + "import helper\n", + "helper.py": "import c2\n", + }, + ) + + assert _entries(_walk(direct)) == ["b.py"] + assert _entries(_walk(through)) == ["c1.py"] + + +def test_a_client_beside_a_stronger_agent_is_not_listed(tmp_path: Path) -> None: + _write( + tmp_path, + { + "pyproject.toml": "", + "app/graph.py": GRAPH, + "tools/backfill.py": CLIENT, + "other/pyproject.toml": "", + "other/bot.py": CLIENT, + }, + ) + + result = _walk(tmp_path) + + assert sorted(_entries(result)) == ["app/graph.py", "other/bot.py"] + + +def test_a_root_test_file_is_side(tmp_path: Path) -> None: + _write( + tmp_path, + {"test_agent.py": GRAPH, "agent_test.py": GRAPH, "conftest.py": GRAPH}, + ) + + result = _walk(tmp_path) + + assert {agent.entry: (agent.side, agent.subfolder) for agent in result.agents} == { + "agent_test.py": (True, ""), + "conftest.py": (True, ""), + "test_agent.py": (True, ""), + } + assert not result.candidate_outside_side + assert not result.source_outside_side + + +def test_the_project_folder_rule(tmp_path: Path) -> None: + one = tmp_path / "one" + _write( + one, + {"pyproject.toml": "", "app/graph.py": GRAPH, "scripts/run.py": GRAPH}, + ) + two = tmp_path / "two" + _write( + two, + { + "svc/pyproject.toml": "", + "svc/a/x.py": GRAPH, + "svc/b/y.py": GRAPH, + "lone/requirements.txt": "", + "lone/deep/z.py": GRAPH, + "plain/graph.py": GRAPH, + }, + ) + + first = _walk(one) + second = _walk(two) + + assert {agent.entry: agent.subfolder for agent in first.agents} == { + "app/graph.py": "", + "scripts/run.py": "scripts", + } + assert ("project pyproject.toml", "pyproject.toml", 0) in _agent( + first, "app/graph.py" + ).evidence + assert {agent.entry: agent.subfolder for agent in second.agents} == { + "svc/a/x.py": "svc/a", + "svc/b/y.py": "svc/b", + "lone/deep/z.py": "lone", + "plain/graph.py": "plain", + } + + +def test_two_graphs_in_one_langgraph_json_mark_the_folder(tmp_path: Path) -> None: + _write( + tmp_path, + { + "svc/pyproject.toml": "", + "svc/langgraph.json": json.dumps( + {"graphs": {"a": "./a.py:graph", "b": "./b.py:graph"}} + ), + "svc/a.py": "graph = None\n", + "svc/b.py": "graph = None\n", + "one/langgraph.json": json.dumps({"graphs": {"a": "./a.py:graph"}}), + "one/a.py": "graph = None\n", + }, + ) + + result = _walk(tmp_path) + + assert [ + (agent.entry, agent.sign, agent.subfolder, agent.build_may_stop) + for agent in result.agents + ] == [ + ("one/a.py", "graph", "one", ""), + ("svc/a.py", "graph", "svc", "graphs"), + ("svc/b.py", "graph", "svc", "graphs"), + ] + assert ("graph", "svc/langgraph.json", 0) in _agent(result, "svc/a.py").evidence + assert build_hazard(tmp_path, "svc") == "graphs" + assert build_hazard(tmp_path, "one") == "" + + +def test_two_agent_py_files_mark_the_folder(tmp_path: Path) -> None: + _write( + tmp_path, + { + "bot/requirements.txt": "", + "bot/agent.py": CLIENT, + "bot/tests/agent.py": "x = 1\n", + "single/agent.py": CLIENT, + "single/node_modules/pkg/agent.py": "", + "single/.venv/agent.py": "", + "single/__pycache__/agent.py": "", + "kept/agent.py": "", + "kept/venv/agent.py": "", + }, + ) + + result = _walk(tmp_path) + + assert { + agent.entry: (agent.sign, agent.subfolder, agent.build_may_stop) + for agent in result.agents + } == { + "bot/agent.py": ("agent_py", "bot", "agent_py"), + "single/agent.py": ("agent_py", "single", ""), + } + assert build_hazard(tmp_path, "kept") == "agent_py" + + +def test_more_than_fifty_agents_in_one_folder_keeps_fifty(tmp_path: Path) -> None: + _write(tmp_path, {f"bot_{index:02d}.py": CLIENT for index in range(51)}) + + over = _walk(tmp_path) + + assert _entries(over) == [f"bot_{index:02d}.py" for index in range(50)] + assert over.more_than_fifty + + (tmp_path / "bot_50.py").unlink() + exact = _walk(tmp_path) + + assert len(exact.agents) == 50 + assert not exact.more_than_fifty + + +def test_paths_his_rule_rejects_are_left_out(tmp_path: Path) -> None: + long_folder = "/".join(["d" * 100] * 5) + _write( + tmp_path, + { + "main.py": GRAPH, + "ok/graph.py": GRAPH, + "my agent/graph.py": GRAPH, + "svc\n/graph.py": GRAPH, + f"{long_folder}/graph.py": GRAPH, + }, + ) + + result = _walk(tmp_path) + + assert {agent.entry: agent.subfolder for agent in result.agents} == { + "main.py": "", + "ok/graph.py": "ok", + } + assert result.paths_left_out == 3 + assert fits_path_rule("ok/graph.py") + assert fits_path_rule("x" * 512) + assert not fits_path_rule("") + assert not fits_path_rule("svc\n") + assert not fits_path_rule("a/../b") + assert not fits_path_rule("x" * 513) + + +def test_an_upload_keeps_one_agent_at_the_root_counted_for_the_root( + tmp_path: Path, +) -> None: + source = tmp_path / "repo" + _write( + source, + { + "svc/pyproject.toml": "", + "svc/graph.py": GRAPH + "import os\nos.getenv('SVC_KEY')\n", + "main.py": "import os\nos.getenv('ROOT_KEY')\n@tool\ndef search(): ...\n", + "prompts/system.txt": "hello\n", + ".env.example": "TEMPLATE_KEY=x\n", + "tests/test_x.py": "import os\nos.getenv('TEST_KEY')\n", + "other/bot.py": CLIENT, + }, + ) + + uploaded = _walk(source, upload=True) + listed = _walk(source) + + [agent] = uploaded.agents + assert (agent.entry, agent.subfolder) == ("svc/graph.py", "") + assert agent.env_names == ("ROOT_KEY", "SVC_KEY", "TEMPLATE_KEY") + assert agent.tools == 1 + assert agent.prompt_files == 1 + assert agent.build_may_stop == build_hazard(source, "") + assert uploaded.upload_others == 1 + in_folder = _agent(listed, "svc/graph.py") + assert in_folder.subfolder == "svc" + assert in_folder.env_names == ("SVC_KEY", "TEMPLATE_KEY") + assert (in_folder.tools, in_folder.prompt_files) == (0, 0) + + side_only = tmp_path / "side" + _write( + side_only, + { + "scripts/run.py": CLIENT + "import os\nos.getenv('SIDE_KEY')\n", + "main.py": "import os\nos.getenv('MAIN_KEY')\n", + }, + ) + + [side] = _walk(side_only, upload=True).agents + + assert (side.entry, side.subfolder, side.side) == ("scripts/run.py", "", True) + assert side.env_names == ("MAIN_KEY", "SIDE_KEY") + + +def test_env_names_are_checked_and_ordered(tmp_path: Path) -> None: + names = [ + "Z_CODE", + "A_CODE", + "BAD NAME", + "ENDS\\n", + "1DIGIT", + "X" * 129, + "Y" * 128, + "A_CODE", + "naïve", + ] + code = "import os\n" + CLIENT + "".join(f"os.getenv('{n}')\n" for n in names) + _write( + tmp_path, + { + "svc/pyproject.toml": "", + "svc/bot.py": code, + "svc/.env.example": "B_TEMPLATE=1\nexport A_TEMPLATE=2\nA_CODE=3\n", + ".env.sample": "ROOT_TEMPLATE=1\n", + "elsewhere/.env.template": "ELSEWHERE=1\n", + }, + ) + + [agent] = _walk(tmp_path).agents + + assert agent.subfolder == "svc" + assert agent.env_names == ( + "A_CODE", + "Y" * 128, + "Z_CODE", + "A_TEMPLATE", + "B_TEMPLATE", + "ROOT_TEMPLATE", + ) + + +def test_describe_entry_reads_process_env(tmp_path: Path) -> None: + _write( + tmp_path, + { + "web/package.json": "{}\n", + "web/src/agent.ts": "const a = process.env.OPENAI_API_KEY;\n" + 'const b = process.env["OTHER_KEY"];\n' + "const c = process.env['THIRD_KEY'];\n" + "const d = process.env[name];\n" + "const e = myprocess.env.NOT_THIS;\n", + "web/src/util.mjs": "export const u = process.env.UTIL_KEY;\n", + "web/tests/agent.test.ts": "process.env.TEST_ONLY;\n", + "web/node_modules/pkg/index.js": "process.env.DEPENDENCY_KEY;\n", + "web/.env.example": "TEMPLATE_KEY=1\n", + "api/server.js": "process.env.ELSEWHERE;\n", + }, + ) + + candidate = describe_entry(tmp_path, "web/src/agent.ts", "web", deadline=FOREVER) + + assert candidate is not None + assert (candidate.sign, candidate.subfolder, candidate.side, candidate.partial) == ( + "client", + "web", + False, + False, + ) + assert candidate.env_names == ( + "OPENAI_API_KEY", + "OTHER_KEY", + "THIRD_KEY", + "UTIL_KEY", + "TEMPLATE_KEY", + ) + + +def test_describe_entry_keeps_the_models_folder_or_finds_package_json( + tmp_path: Path, +) -> None: + source = tmp_path / "repo" + _write( + source, + { + "pyproject.toml": "", + "web/package.json": "{}\n", + "web/src/agent.ts": "export {};\n", + "plain/deep/bot.py": CLIENT, + }, + ) + bare = tmp_path / "bare" + _write(bare, {"a/b/bot.py": CLIENT}) + + def folder_of(root: Path, entry: str, folder: str, upload: bool = False) -> str: + found = describe_entry(root, entry, folder, deadline=FOREVER, upload=upload) + assert found is not None + return found.subfolder + + assert folder_of(source, "web/src/agent.ts", "web/src") == "web/src" + assert folder_of(source, "web/src/agent.ts", "") == "" + assert folder_of(source, "web/src/agent.ts", "elsewhere") == "web" + assert folder_of(source, "web/src/agent.ts", "web/sr") == "web" + assert folder_of(source, "web/src/agent.ts", "web\n") == "web" + assert folder_of(source, "plain/deep/bot.py", "nowhere") == "" + assert folder_of(source, "web/src/agent.ts", "web/src", upload=True) == "" + assert folder_of(bare, "a/b/bot.py", "x") == "a/b" + assert describe_entry(source, "web/src/agent.ts\n", "web", deadline=FOREVER) is None + assert describe_entry(source, "", "", deadline=FOREVER) is None + + +def test_describe_entry_stops_at_its_deadline_with_partial( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + assert (scan_walk._DESCRIBE_FILES, scan_walk._DESCRIBE_BYTES) == (2_000, 33_554_432) + _write( + tmp_path, + { + "web/package.json": "{}\n", + "web/agent.ts": "process.env.FIRST;\n", + "web/b.ts": "process.env.SECOND;\n", + "web/c.ts": "process.env.THIRD;\n", + }, + ) + + late = describe_entry( + tmp_path, "web/agent.ts", "web", deadline=0.0, clock=lambda: 1.0 + ) + on_time = describe_entry( + tmp_path, "web/agent.ts", "web", deadline=10.0, clock=lambda: 1.0 + ) + monkeypatch.setattr(scan_walk, "_DESCRIBE_FILES", 1) + capped = describe_entry(tmp_path, "web/agent.ts", "web", deadline=FOREVER) + + assert late is not None and on_time is not None and capped is not None + assert (late.entry, late.partial, late.env_names) == ("web/agent.ts", True, ()) + assert not on_time.partial + assert on_time.env_names == ("FIRST", "SECOND", "THIRD") + assert (capped.partial, capped.env_names) == (True, ("FIRST",)) + + +# Every value line below carries "FAKE", so any name holding it is a piece of a value. +FAKE_KEY_LINE = "FAKEk3yAAAAAbbbbbbbbCCCCCCCCdddddddd0123456789EEEEEEEEffffffffGG" +AROUND = ("A_BEFORE", "Z_AFTER") +# Each shape: a settings template, and the names a reader of it should give, no more. +TEMPLATE_SHAPES: dict[str, tuple[str, tuple[str, ...]]] = { + "wrapped_bare_short_tail": ( + f"A_BEFORE=1\nSIGNING_KEY={FAKE_KEY_LINE}\n{FAKE_KEY_LINE}\nFAKEt4il=\nZ_AFTER=2\n", + (*AROUND, "SIGNING_KEY"), + ), + "wrapped_once_short_tail": ( + f"A_BEFORE=1\nSIGNING_KEY={FAKE_KEY_LINE}\nFAKEt4il=\nZ_AFTER=2\n", + (*AROUND, "SIGNING_KEY"), + ), + "bare_body_below_an_empty_setting": ( + f"A_BEFORE=1\nSIGNING_KEY=\n{FAKE_KEY_LINE}\nFAKEt4il=\nZ_AFTER=2\n", + (*AROUND, "SIGNING_KEY"), + ), + "crlf_wrapped_short_tail": ( + f"A_BEFORE=1\r\nKEY={FAKE_KEY_LINE}\r\nFAKEcb=\r\nZ_AFTER=2\r\n", + (*AROUND, "KEY"), + ), + "base64url_tail_with_underscore": ( + f"A_BEFORE=1\nJWK_D={FAKE_KEY_LINE}\nFAKE_url_tail_AAAAbbbbCCCCddddEEEEffff0123=\nZ_AFTER=2\n", + (*AROUND, "JWK_D"), + ), + "base64url_line_below_an_empty_setting": ( + "A_BEFORE=1\nJWK_D=\nFAKE_url_tail_AAAAbbbbCCCCddddEEEEffff0123=\nZ_AFTER=2\n", + (*AROUND, "JWK_D"), + ), + "base32_tail": ( + "A_BEFORE=1\nTOTP_SEED=FAKEBASE32AAAABBBBCCCCDDDDEEEE\nFAKEBASE32TAIL234567ABCDEFGH=\nZ_AFTER=2\n", + (*AROUND, "TOTP_SEED"), + ), + "triple_double_quotes": ( + f'A_BEFORE=1\nTRIPLE="""\n{FAKE_KEY_LINE}\nFAKEtq=\n"""\nZ_AFTER=2\n', + (*AROUND, "TRIPLE"), + ), + "triple_single_quotes": ( + f"A_BEFORE=1\nTRIPLE='''\n{FAKE_KEY_LINE}\nFAKEts=\n'''\nZ_AFTER=2\n", + (*AROUND, "TRIPLE"), + ), + "escaped_single_quote": ( + f"A_BEFORE=1\nESC='it\\'s\n{FAKE_KEY_LINE}\nFAKEesc=\n'\nZ_AFTER=2\n", + (*AROUND, "ESC"), + ), + "escaped_double_quote": ('A_BEFORE=1\nESC="say \\"hi\nFAKEesd=\n"\nZ_AFTER=2\n', (*AROUND, "ESC")), + "quoted_value_left_open": ('A_BEFORE=1\nKEY="\nFAKEdq=\n"\nZ_AFTER=2\n', (*AROUND, "KEY")), + "key_block_with_one_short_line": ( + "A_BEFORE=1\nPEM=-----BEGIN PRIVATE KEY-----\nFAKEpm=\n-----END PRIVATE KEY-----\nZ_AFTER=2\n", + (*AROUND, "PEM"), + ), + "padded_tail_below_a_short_line": ( + "A_BEFORE=1\nKEY=FAKEshortline\nFAKEpad==\nZ_AFTER=2\n", + (*AROUND, "KEY"), + ), + "key_block_opened_in_a_comment": ( + f"A_BEFORE=1\n#PEM=-----BEGIN PRIVATE KEY-----\n{FAKE_KEY_LINE}\nFAKEpc=\n" + "-----END PRIVATE KEY-----\nZ_AFTER=2\n", + AROUND, + ), + "line_joined_by_a_backslash": ( + f"A_BEFORE=1\nexport LONG={FAKE_KEY_LINE}\\\nFAKEbs=\nZ_AFTER=2\n", + (*AROUND, "LONG"), + ), + "lines_joined_by_backslashes": ( + "A_BEFORE=1\nexport OPTS=--first \\\n --second \\\n FAKEthird=x\nZ_AFTER=2\n", + (*AROUND, "OPTS"), + ), + "key_wrapped_below_a_joined_line": ( + f"A_BEFORE=1\nexport LONG={FAKE_KEY_LINE}\\\n{FAKE_KEY_LINE}\nFAKEt4il=\nZ_AFTER=2\n", + (*AROUND, "LONG"), + ), + "line_separator_in_a_value": ("A_BEFORE=1\nKEY=abc\u2028FAKEls=\nZ_AFTER=2\n", (*AROUND, "KEY")), + "form_feed_in_a_value": ("A_BEFORE=1\nKEY=abc\x0cFAKEff=\nZ_AFTER=2\n", (*AROUND, "KEY")), + "byte_order_mark": ("\ufeffOPENAI_API_KEY=\nZ_AFTER=2\n", ("OPENAI_API_KEY", "Z_AFTER")), + "escaped_backslash_before_the_closing_quote": ( + 'A_BEFORE=1\nWIN="C:\\\\tmp\\\\"\nMID_NAME=3\nZ_AFTER=2\n', + (*AROUND, "WIN", "MID_NAME"), + ), + "key_marker_in_an_inline_comment": ( + "A_BEFORE=1\nKEY=x # paste -----BEGIN RSA PRIVATE KEY----- here\nMID_NAME=3\nZ_AFTER=2\n", + (*AROUND, "KEY", "MID_NAME"), + ), + "key_marker_in_a_comment_line": ( + "A_BEFORE=1\n# paste the key from -----BEGIN on\nMID_NAME=3\nZ_AFTER=2\n", + (*AROUND, "MID_NAME"), + ), + "one_line_key_block": ( + "A_BEFORE=1\nKEY=-----BEGIN FAKE KEY-----FAKEbody-----END FAKE KEY-----\nMID_NAME=3\nZ_AFTER=2\n", + (*AROUND, "KEY", "MID_NAME"), + ), + "word_names": ( + "apiKey=\njwtSecret=\nOPENAIAPIKEY=\nlowercasename=\nLANGCHAIN_TRACING_V2=\n" + "NEXT_PUBLIC_SUPABASE_ANON_KEY=\n", + ( + "apiKey", + "jwtSecret", + "OPENAIAPIKEY", + "lowercasename", + "LANGCHAIN_TRACING_V2", + "NEXT_PUBLIC_SUPABASE_ANON_KEY", + ), + ), + "export_quoted": ( + f'A_BEFORE=1\nexport KEY="{FAKE_KEY_LINE}\nFAKEex=\n"\nZ_AFTER=2\n', + (*AROUND, "KEY"), + ), + "inline_comment_with_a_quote": ( + "A_BEFORE=1\nKEY=plain # don't\nMID_NAME=3\nKEY2=\"v\" # it's\nZ_AFTER=2\n", + (*AROUND, "KEY", "MID_NAME", "KEY2"), + ), + "key_block_marked": ( + f"A_BEFORE=1\nPEM=-----BEGIN PRIVATE KEY-----\n{FAKE_KEY_LINE}\nFAKEpm=\n" + "-----END PRIVATE KEY-----\nZ_AFTER=2\n", + (*AROUND, "PEM"), + ), + "json_blob_over_lines": ( + 'A_BEFORE=1\nSA={\n "type": "service_account",\n "private_key_id": "FAKEid",\n' + ' "private_key": "-----BEGIN PRIVATE KEY-----\\nFAKEjs=\\n-----END PRIVATE KEY-----\\n"\n' + "}\nZ_AFTER=2\n", + (*AROUND, "SA"), + ), + "json_blob_on_one_line": ( + 'A_BEFORE=1\nSA={"k":"FAKEjc=","p":"FAKEjc2="}\nZ_AFTER=2\n', + (*AROUND, "SA"), + ), + "crlf_quoted": ( + f'A_BEFORE=1\r\nKEY="{FAKE_KEY_LINE}\r\nFAKEcr=\r\n"\r\nZ_AFTER=2\r\n', + (*AROUND, "KEY"), + ), + "backtick_over_two_lines": ( + f"A_BEFORE=1\nKEY=`{FAKE_KEY_LINE}\nFAKEbt=`\nZ_AFTER=2\n", + (*AROUND, "KEY"), + ), + "backtick_over_three_lines": ( + f"A_BEFORE=1\nKEY=`{FAKE_KEY_LINE}\n{FAKE_KEY_LINE}\nFAKEbt=`\nMID_NAME=3\nZ_AFTER=2\n", + (*AROUND, "KEY", "MID_NAME"), + ), + "escaped_backtick": ("A_BEFORE=1\nESC=`say \\`hi\nFAKEebt=\n`\nZ_AFTER=2\n", (*AROUND, "ESC")), + "encoded_line_ending_in_a_blank": ( + f"A_BEFORE=1\nKEY={FAKE_KEY_LINE} \nFAKEtb=\nZ_AFTER=2\n", + (*AROUND, "KEY"), + ), + "encoded_line_ending_in_a_tab": ( + f"A_BEFORE=1\nKEY={FAKE_KEY_LINE}\t\nFAKEtt=\nZ_AFTER=2\n", + (*AROUND, "KEY"), + ), + "encoded_line_ending_in_a_comment": ( + f"A_BEFORE=1\nKEY={FAKE_KEY_LINE} # paste the key here\nFAKEtc=\nZ_AFTER=2\n", + (*AROUND, "KEY"), + ), +} + + +def _name_problems(names: object, expected: tuple[str, ...]) -> tuple[list[int], list[str]]: + """The lengths of the names that shouldn't be there, and the names missing. Lengths only, + so a failure never prints a piece of a value.""" + found = set(names) # type: ignore[call-overload] + return sorted(len(name) for name in found - set(expected)), sorted(set(expected) - found) + + +@pytest.mark.parametrize("shape", sorted(TEMPLATE_SHAPES)) +def test_a_template_shape_gives_its_names_and_no_value(shape: str) -> None: + text, expected = TEMPLATE_SHAPES[shape] + + problems = _name_problems(scan_walk._template_names(text.encode("utf-8")), expected) + + assert problems == ([], []) + + +@pytest.mark.parametrize( + "encoded", + [ + lambda text: b"\xff\xfe" + text.encode("utf-16-le"), + lambda text: b"\xfe\xff" + text.encode("utf-16-be"), + lambda text: text.encode("latin-1"), + ], + ids=["utf16_little_endian", "utf16_big_endian", "latin1"], +) +def test_a_template_in_utf16_or_latin1_gives_its_names_and_no_value( + tmp_path: Path, encoded: object +) -> None: + # Not UTF-8 either way: a byte-order mark of UTF-16, or a Latin-1 `ü`. + text = ( + f"A_BEFORE=1\nCITY=Zürich\nSIGNING_KEY={FAKE_KEY_LINE}\n{FAKE_KEY_LINE}\nFAKEt4il=\n" + "Z_AFTER=2\n" + ) + expected = ("A_BEFORE", "CITY", "SIGNING_KEY", "Z_AFTER") + data = encoded(text) # type: ignore[operator] + (tmp_path / ".env.example").write_bytes(data) + (tmp_path / "bot.py").write_text(CLIENT, encoding="utf-8") + + problems = _name_problems(scan_walk._template_names(data), expected) + walked = _walk(tmp_path) + walked_problems = _name_problems(walked.agents[0].env_names, expected) + + assert problems == ([], []) + assert walked.unread_files == 0 + assert walked_problems == ([], []) + + +def test_a_multi_line_value_in_a_template_is_never_a_name(tmp_path: Path) -> None: + # Every line of a pasted key below carries "Fake"; the last base64 line of a key ends in + # `=` padding, so it reads like a setting with an empty value. + template = ( + "BEFORE_NAME=1\n" + 'QUOTED_KEY="-----BEGIN PRIVATE KEY-----\n' + "MIIEvQIBADANBgkqhkiG9w0BAQEFAASCFake01\n" + "QuotedTailFake02=\n" + "-----END PRIVATE KEY-----\"\n" + "MIDDLE_NAME=2\n" + "RAW_KEY=-----BEGIN PRIVATE KEY-----\n" + "MIIEvQIBADANBgkqhkiG9w0BAQEFAASCFake03\n" + "RawTailFake04=\n" + "-----END PRIVATE KEY-----\n" + "SINGLE_QUOTED='first line\n" + "SingleTailFake05=\n" + "'\n" + "BarePaddedFake06==\n" + "WRAPPED_BLOB=MIIBIjANBgkqhkiG9w0BAQEFAAOCFake07\n" + "hkiG9w0BAQEFAAOCAQ8AFake08=\n" + "AFTER_NAME=3\n" + "OPENAIAPIKEY=\n" + "lowercasename=\n" + ) + expected = ( + "AFTER_NAME", + "BEFORE_NAME", + "MIDDLE_NAME", + "OPENAIAPIKEY", + "QUOTED_KEY", + "RAW_KEY", + "SINGLE_QUOTED", + "WRAPPED_BLOB", + "lowercasename", + ) + # Each shape in a repository of its own, so one shape's value can't hide another's. + repos = {"mixed": (template, expected), **TEMPLATE_SHAPES} + for shape, (text, _) in repos.items(): + (tmp_path / shape).mkdir() + (tmp_path / shape / ".env.example").write_bytes(text.encode("utf-8")) + (tmp_path / shape / "bot.py").write_text(CLIENT, encoding="utf-8") + + for shape, (_, names) in repos.items(): + [agent] = _walk(tmp_path / shape).agents + described = describe_entry(tmp_path / shape, "bot.py", "", deadline=FOREVER) + + assert described is not None + for found in (agent.env_names, described.env_names): + problems = _name_problems(found, names) + assert (shape, problems) == (shape, ([], [])) + assert found == tuple(sorted(names)) + assert not any("FAKE" in name.upper() or "MIIE" in name for name in found), shape + + +def test_the_walks_hazard_look_stops_at_the_walks_deadline( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # The build counts `agent.py` in `venv/`, which the walk itself skips. + _write(tmp_path, {"bot.py": CLIENT, "venv/a/agent.py": "", "venv/b/agent.py": ""}) + [on_time] = _walk(tmp_path).agents + assert on_time.build_may_stop == "agent_py" + listed: list[str] = [] + real_walk = os.walk + + def counting(top, *args, **kwargs): + for directory, directories, files in real_walk(top, *args, **kwargs): + listed.extend(os.path.join(directory, name) for name in directories + files) + yield directory, directories, files + + monkeypatch.setattr(scan_walk.os, "walk", counting) + # On time while the walk reads its one file, past the deadline from then on. + ticks = iter([0.0]) + + [late] = walk_repo(tmp_path, deadline=10.0, clock=lambda: next(ticks, 100.0)).agents + + assert (late.entry, late.build_may_stop) == ("bot.py", "") + assert not any(path.endswith("agent.py") for path in listed) + + +def test_build_hazard_is_bounded_and_asked_once_per_folder( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + assert scan_walk._HAZARD_ENTRIES == 200_000 + _write(tmp_path, {f"bot_{index}.py": CLIENT for index in range(5)}) + for index in range(30): + (tmp_path / "venv" / f"pkg_{index:02d}").mkdir(parents=True) + entries: list[str] = [] + real_walk = os.walk + + def counting(top, *args, **kwargs): + for directory, directories, files in real_walk(top, *args, **kwargs): + entries.extend(os.path.join(directory, name) for name in directories + files) + yield directory, directories, files + + monkeypatch.setattr(scan_walk.os, "walk", counting) + venv = os.path.join(tmp_path, "venv") + os.sep + + hazards: dict[str, str] = {} + result = walk_repo(tmp_path, deadline=FOREVER, hazards=hazards) + for _ in range(2): + describe_entry(tmp_path, "bot_0.py", "", deadline=FOREVER, hazards=hazards) + + # Five root agents and two descriptions share the root: its skipped `venv/` is looked + # through once. + assert len(result.agents) == 5 + assert hazards == {"": ""} + assert sum(path.startswith(venv) for path in entries) == 30 + + # Two `agent.py` files, each 15 folders down: finding both visits about 30 entries. + for branch in ("a", "b"): + _write(tmp_path, {"/".join(["deep", *[branch] * 15, "agent.py"]): ""}) + entries.clear() + monkeypatch.setattr(scan_walk, "_HAZARD_ENTRIES", 10) + assert build_hazard(tmp_path, "deep") == "" + assert len(entries) <= 11 + + monkeypatch.setattr(scan_walk, "_HAZARD_ENTRIES", 200_000) + assert build_hazard(tmp_path, "deep") == "agent_py" + + entries.clear() + assert build_hazard(tmp_path, "deep", deadline=0.0, clock=lambda: 1.0) == "" + # The folder's own listing and the one the clock is checked after, never the rest. + assert len(entries) <= 3 + + +def test_the_prompt_files_listed_are_capped( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + assert scan_walk._MAX_PROMPT_PATHS == 20_000 + _write(tmp_path, {"bot.py": CLIENT, **{f"prompts/p{index}.txt": "x\n" for index in range(5)}}) + monkeypatch.setattr(scan_walk, "_MAX_PROMPT_PATHS", 3) + + [agent] = _walk(tmp_path).agents + + assert agent.prompt_files == 3 + + +def _decision_table_sources(row: str) -> tuple[str, ...]: + """One row's files from the framework check's own decision table, read from its test.""" + tree = ast.parse((Path(__file__).parent / "test_framework_detection.py").read_text()) + test = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) and node.name == "test_decision_table" + ) + table = test.decorator_list[0] + assert isinstance(table, ast.Call) + ids = next(keyword.value for keyword in table.keywords if keyword.arg == "ids") + index = ast.literal_eval(ids).index(row) + rows = table.args[1] + assert isinstance(rows, ast.List) + sources, _ = rows.elts[index].elts # type: ignore[attr-defined] + return ast.literal_eval(sources) + + +def test_a_chat_model_alone_is_a_candidate(tmp_path: Path) -> None: + [source] = _decision_table_sources("model_alone") + _write(tmp_path, {"bot.py": source}) + + result = _walk(tmp_path) + + [agent] = result.agents + assert (agent.entry, agent.sign, agent.builds) == ("bot.py", "model", ()) + assert result.candidate_outside_side + assert ("model langchain", "bot.py", 2) in agent.evidence + + +def test_a_chat_model_beside_a_stronger_agent_is_not_listed(tmp_path: Path) -> None: + chat_model = "from langchain_openai import ChatOpenAI\nChatOpenAI()\n" + _write( + tmp_path, + { + "svc/pyproject.toml": "", + "svc/graph.py": GRAPH, + "svc/helper.py": chat_model, + "other/pyproject.toml": "", + "other/chain.py": chat_model, + }, + ) + + result = _walk(tmp_path) + + assert {agent.entry: agent.sign for agent in result.agents} == { + "svc/graph.py": "framework", + "other/chain.py": "model", + } From 159c580741e2441fe2d080ecd2272a5975fc00b5 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Fri, 6 Nov 2026 13:58:20 +0530 Subject: [PATCH 03/23] fix(harness): keep job input out of a reading's failure, and share what it copied - A job that fails validation names only where and how, never its values, so a secret in the job can't reach failure.json. - Swallowed errors and crashes log a warning with their trace, masked. - The reading's words, the refusal exit, the folder choice and the cost writer are shared with the build instead of copied. - A stage takes its own idle retries; the time limits are worked out when a reading starts; the timer stops with the work. - Classifying a reading's result moves to reading_result.py. TH-8951 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_reading.py | 369 ++++----------------- src/fi/alk/harness/authoring_entrypoint.py | 8 + src/fi/alk/harness/cli.py | 7 +- src/fi/alk/harness/framework_detection.py | 22 +- src/fi/alk/harness/reading_entry.py | 185 +++++++---- src/fi/alk/harness/reading_result.py | 250 ++++++++++++++ src/fi/alk/harness/session.py | 8 +- src/fi/alk/harness/spend.py | 39 ++- src/fi/alk/harness/understand.py | 5 + tests/harness/test_agent_reading.py | 184 ++++++---- tests/harness/test_reading_entry.py | 160 ++++++++- 11 files changed, 774 insertions(+), 463 deletions(-) create mode 100644 src/fi/alk/harness/reading_result.py diff --git a/src/fi/alk/harness/agent_reading.py b/src/fi/alk/harness/agent_reading.py index 1678aa96a..1b929a739 100644 --- a/src/fi/alk/harness/agent_reading.py +++ b/src/fi/alk/harness/agent_reading.py @@ -10,50 +10,49 @@ from __future__ import annotations import asyncio -import json +import functools import logging -import math -import os -import re -import tempfile import time from dataclasses import replace from pathlib import Path from typing import Any -from .. import __version__ -from ..providers.agent import AgentBuildError, build_authored_agent from . import session +from .agent_type_check import EXIT_REFUSED from .backends import StageDone from .builder import BUILDER_TURNS from .config import chosen_model -from .contract import AgentContract from .failure_file import FAILURE_FILE -from .framework_detection import _contract_workdir, detect_framework -from .job import FailureDomain, HarnessFailure, HarnessJob, HarnessStage, SourceKind +from .job import FailureDomain, HarnessFailure, HarnessJob, SourceKind from .reading_entry import ( + EXIT_RETRYABLE, READING_JOB_INVALID, + MaskSecrets, ReadingMode, + limit_seconds, + reading_failure, write_reading_cost, write_reading_failure, ) +from .reading_result import ( + CONTRACT_FILE, + _classify, + _document, + _model_error, + _out_of_turns, + _saved, +) from .session import Stage, StageIdleTimeout -from .source_folder import agent_folder, refuse_unusable_folder +from .source_folder import refuse_unusable_folder from .sources import resolve as resolve_source -from .understand import open_stage, opening +from .understand import SUBMIT_REMINDER, open_stage, opening logger = logging.getLogger(__name__) - -AGENT_FILE = "agent.json" -CONTRACT_FILE = "contract.json" +logger.addFilter(MaskSecrets()) READING_SOURCE_EMPTY = "reading_source_empty" AGENT_READING_TIMED_OUT = "agent_reading_timed_out" AGENT_READING_CRASHED = "agent_reading_crashed" -AGENT_READING_MODEL_ERROR = "agent_reading_model_error" -AGENT_READING_INCOMPLETE = "agent_reading_incomplete" -AGENT_READING_TOO_LARGE = "agent_reading_too_large" -AGENT_DOCUMENT_INVALID = "agent_document_invalid" _LIMIT_ENV = "ALK_READING_MAX_DURATION_SECONDS" _DEFAULT_SOFT_LIMIT_SECONDS = 1800.0 @@ -63,59 +62,18 @@ _INTERRUPT_SECONDS = 10.0 _INTERRUPT_WAIT_SECONDS = 60.0 -# The platform's own caps, compared the way it compares them. -_DOCUMENT_LIMIT_BYTES = 1_048_576 -_CONTRACT_LIMIT_BYTES = 5 * 1_048_576 - -# A model out of turns can't save, so a reminder after one only adds cost. -_TURN_LIMITS = frozenset({"max_turns", "error_max_turns"}) -# Matched only, never repeated in a message: these are ADK's words, not ours. -_CALL_LIMIT = re.compile(r"Max number of llm calls limit of `\d+` exceeded") -# Statuses a retry may get past; any other 4xx would only be refused again. -_PASSING_STATUSES = frozenset({408, 409, 429}) _COUNTS = ("tokens_in", "tokens_out", "tokens_cached") -_EXIT_DONE = 0 -_EXIT_RETRYABLE = 1 -_EXIT_REFUSED = 78 - -# The same words the build's unattended reading uses, so the two readings can't drift apart. -_REMINDER = ( - "Nothing was saved: you finished without calling submit_contract. Call it now " - "with the contract you worked out." -) - def _soft_limit() -> float: - try: - seconds = float(os.environ.get(_LIMIT_ENV, "")) - except ValueError: - return _DEFAULT_SOFT_LIMIT_SECONDS - return seconds if math.isfinite(seconds) and seconds > 0 else _DEFAULT_SOFT_LIMIT_SECONDS - - -def _failure( - domain: FailureDomain, - code: str, - message: str, - *, - retryable: bool, - details: dict[str, Any] | None = None, -) -> HarnessFailure: - return HarnessFailure( - domain=domain, - stage=HarnessStage.UNDERSTANDING_AGENT, - code=code, - message=message, - retryable=retryable, - details=details or {}, - ) + # The helper reads the same variable the same way for its own limit. + return limit_seconds(_LIMIT_ENV, _DEFAULT_SOFT_LIMIT_SECONDS) def _check_job(job: HarnessJob) -> HarnessFailure | None: if job.source.kind in (SourceKind.GITHUB, SourceKind.ARCHIVE): return None - return _failure( + return reading_failure( FailureDomain.PLATFORM_SYNC, READING_JOB_INVALID, f"An agent is read from a GitHub repository or an upload, not from a " @@ -125,7 +83,7 @@ def _check_job(job: HarnessJob) -> HarnessFailure | None: class _ReadingStage(Stage): - """The build's reading stage, with each turn's own spend kept and written as it lands.""" + """The build's reading stage, with each turn's own spend kept, and no turn ever replayed.""" def __init__( self, @@ -136,7 +94,10 @@ def __init__( output: Path, running_totals: bool, ) -> None: - super().__init__(spec, name=name, backend=backend) + # Never replayed after a quiet stream: a replayed turn is paid for again, and a new + # session would restart Claude's running totals under the baseline the difference is + # taken from. + super().__init__(spec, name=name, backend=backend, idle_retries=0) self.output = output self.running_totals = running_totals self.done: list[StageDone] = [] @@ -172,8 +133,6 @@ def _events(self, received: Any, turn: Any) -> list[Any]: events = super()._events(done, turn) self.done.append(done) self.in_flight = False - # On disk after every turn, so a sandbox that dies mid-reading still leaves what it spent. - write_reading_cost(self.output, complete=False) return events async def say(self, message: str, **kwargs: Any) -> Any: @@ -181,7 +140,7 @@ async def say(self, message: str, **kwargs: Any) -> Any: return await super().say(message, **kwargs) -def _holds_anything(source: Path) -> bool: +def _source_has_files(source: Path) -> bool: try: return source.is_dir() and next(source.iterdir(), None) is not None except OSError: @@ -189,26 +148,6 @@ def _holds_anything(source: Path) -> bool: return False -def _saved(output: Path) -> str: - if (output / FAILURE_FILE).is_file(): - return FAILURE_FILE - if (output / CONTRACT_FILE).is_file(): - return CONTRACT_FILE - return "" - - -def _out_of_turns(done: StageDone) -> bool: - # ADK enforces its call budget by raising, which the Gemini backend reports as a failure with - # no status, so ADK's own words are the signal. Matched whole, since the phrase inside other - # text may be the model's own. The count is no signal: it catches no real limit, and it would - # read a request cut on the last allowed call as one. - return done.outcome in _TURN_LIMITS or ( - done.is_error - and done.api_error_status is None - and any(_CALL_LIMIT.fullmatch(str(said)) for said in done.errors) - ) - - def _reminder_due(stage: _ReadingStage, output: Path, stopping: asyncio.Event) -> bool: if _saved(output) or stopping.is_set(): return False @@ -226,53 +165,33 @@ async def _converse( async with stage: await stage.say(opening(agent_source)) if _reminder_due(stage, output, stopping): - await stage.say(_REMINDER) + await stage.say(SUBMIT_REMINDER) -async def _stop(stage: _ReadingStage, task: asyncio.Task[None], stopping: asyncio.Event) -> None: +async def _stop( + stage: _ReadingStage, task: asyncio.Task[None], stopping: asyncio.Event, wait: float +) -> None: # Set first, so a turn ending now doesn't earn the reminder. stopping.set() try: await asyncio.wait_for(stage.interrupt_response(), _INTERRUPT_SECONDS) except Exception: # A stop that can't be delivered is still a timeout, not a crash. - pass - await asyncio.wait({task}, timeout=_INTERRUPT_WAIT_SECONDS) + logger.warning("the stop at the time limit was not delivered", exc_info=True) + await asyncio.wait({task}, timeout=wait) # Closed before the files are read, so no late tool call can save after the decision. task.cancel() try: await task - except (asyncio.CancelledError, Exception): + except asyncio.CancelledError: pass - - -def _refusing(status: Any) -> bool: - return ( - isinstance(status, int) - and not isinstance(status, bool) - and 400 <= status <= 499 - and status not in _PASSING_STATUSES - ) - - -def _model_error(output: Path, message: str, status: Any = None, *, retryable: bool) -> int: - write_reading_failure( - _failure( - FailureDomain.INFRASTRUCTURE, - AGENT_READING_MODEL_ERROR, - message, - retryable=retryable, - # Only our own reading of the status: what the provider said stays in the message. - details={"status": status if isinstance(status, int) else None}, - ), - output, - ) - return _EXIT_RETRYABLE if retryable else _EXIT_REFUSED + except Exception: + logger.warning("the stopped reading ended with an error", exc_info=True) def _timed_out(output: Path, soft_limit: float) -> int: write_reading_failure( - _failure( + reading_failure( FailureDomain.INFRASTRUCTURE, AGENT_READING_TIMED_OUT, f"The agent reading ran past its {soft_limit:g} s limit without saving the agent's " @@ -281,176 +200,17 @@ def _timed_out(output: Path, soft_limit: float) -> int: ), output, ) - return _EXIT_REFUSED - - -def _classify( - job: HarnessJob, - source: Path, - output: Path, - stage: _ReadingStage, - error: BaseException | None, -) -> int: - saved = _saved(output) - if saved == FAILURE_FILE: - return _EXIT_REFUSED - # Before any error: the gate accepted the reading, and refusing it now would pay for it twice. - if saved == CONTRACT_FILE: - return _document(job, source, output) - forbidden = stage.forbidden_models() - if forbidden: - # A retry would bill the same model again. - return _model_error( - output, - "The provider billed a model this harness may not spend on (" - + ", ".join(sorted(forbidden)) - + "), so the reading was stopped.", - retryable=False, - ) - # A last turn out of turns spent a whole budget, so a retry would likely spend another and - # stop the same way, whatever failed before it. - ran_out = bool(stage.done) and _out_of_turns(stage.done[-1]) - failed = [ - done - for done in stage.done - if done.api_error_status is not None - or (done.is_error and not _out_of_turns(done)) - ] - if not ran_out and (failed or error is not None): - status = next( - (done.api_error_status for done in reversed(failed) if done.api_error_status is not None), - None, - ) - said = [str(one) for done in failed for one in done.errors] - if error is not None: - said.append(f"{type(error).__name__}: {error}") - shown = f" (status {status})" if isinstance(status, int) else "" - return _model_error( - output, - f"The model provider failed while the agent was being read{shown}: " - + ("; ".join(said) or "no detail was given."), - status, - retryable=not _refusing(status), - ) - write_reading_failure( - _failure( - FailureDomain.INFRASTRUCTURE, - AGENT_READING_INCOMPLETE, - "The model finished without saving the agent's description.", - retryable=False, - details={ - "outcome": stage.done[-1].outcome if stage.done else "", - "turns": sum(int(done.turns or 0) for done in stage.done), - }, - ), - output, - ) - return _EXIT_REFUSED - - -class _TooLarge(Exception): - def __init__(self, sizes: dict[str, int]) -> None: - super().__init__("too large") - self.sizes = sizes - - -def _framework_folder(job: HarnessJob, source: Path, output: Path) -> Path: - # As a build chooses it when there is no build: the named folder, else the contract's own. - if job.source.subfolder: - return agent_folder(source, job.source.subfolder) - return _contract_workdir(source, output) or source - - -def _write_whole(path: Path, body: bytes) -> None: - # Written whole, then moved: the platform must never read half a document. - handle = tempfile.NamedTemporaryFile( - "wb", dir=path.parent, prefix=".agent-", suffix=".tmp", delete=False - ) - try: - with handle as writing: - writing.write(body) - os.replace(handle.name, path) - except BaseException: - Path(handle.name).unlink(missing_ok=True) - raise - - -def _document(job: HarnessJob, source: Path, output: Path) -> int: - """The agent's description from the saved contract, as a build makes it.""" - # One catch over every step: a saved contract is paid for, so nothing after it is retryable. - try: - contract_bytes = (output / CONTRACT_FILE).stat().st_size - if contract_bytes > _CONTRACT_LIMIT_BYTES: - raise _TooLarge({"contract_bytes": contract_bytes}) - contract = AgentContract.model_validate( - json.loads((output / CONTRACT_FILE).read_text(encoding="utf-8")) - ).model_dump(mode="json") - # The echo proves which folder was read, so it comes from the contract. - built_from = str((contract.get("runtime") or {}).get("source_subfolder") or "") - if built_from != (job.source.subfolder or ""): - raise AgentBuildError("subfolder_mismatch") - framework = detect_framework( - _framework_folder(job, source, output), repo_root=source, package_dir=None - ) - inner = build_authored_agent( - contract, subfolder=built_from or None, framework=framework - ) - body = json.dumps( - {"sdk_version": __version__, "agent": inner}, - ensure_ascii=False, - separators=(",", ":"), - ).encode("utf-8") - stored = json.dumps(inner, ensure_ascii=False).encode("utf-8") - if len(body) > _DOCUMENT_LIMIT_BYTES or len(stored) > _DOCUMENT_LIMIT_BYTES: - raise _TooLarge( - { - "agent_bytes": len(body), - "stored_bytes": len(stored), - "contract_bytes": contract_bytes, - } - ) - _write_whole(output / AGENT_FILE, body) - except _TooLarge as too_large: - write_reading_failure( - _failure( - FailureDomain.INFRASTRUCTURE, - AGENT_READING_TOO_LARGE, - "The agent's description is larger than the platform stores (1 MiB for the " - "description, 5 MiB for the contract).", - retryable=False, - details=dict(too_large.sizes), - ), - output, - ) - return _EXIT_REFUSED - except Exception as exc: - code = str(getattr(exc, "code", "") or "") - logger.warning( - "agent document not built: class=%s code=%s", type(exc).__name__, code - ) - write_reading_failure( - _failure( - FailureDomain.INFRASTRUCTURE, - AGENT_DOCUMENT_INVALID, - "The agent's description could not be built from its saved contract " - f"({type(exc).__name__}{': ' + code if code else ''}).", - retryable=False, - details={"class": type(exc).__name__, "code": code}, - ), - output, - ) - return _EXIT_REFUSED - return _EXIT_DONE + return EXIT_REFUSED async def _reading( - job: HarnessJob, source: Path, output: Path, started: float, soft_limit: float + job: HarnessJob, source: Path, output: Path, started: float, soft_limit: float, wait: float ) -> tuple[int, bool]: """The exit code, and whether every turn's spend is known to be in the cost file.""" # Before the folder check, so an upload that hasn't arrived isn't blamed on the folder. - if not _holds_anything(source): + if not _source_has_files(source): write_reading_failure( - _failure( + reading_failure( FailureDomain.PLATFORM_SYNC, READING_SOURCE_EMPTY, "The agent's source was missing or empty, so there was nothing to read. " @@ -459,13 +219,10 @@ async def _reading( ), output, ) - return _EXIT_RETRYABLE, True + return EXIT_RETRYABLE, True if refuse_unusable_folder(source, job.source.subfolder, output): - return _EXIT_REFUSED, True + return EXIT_REFUSED, True - # A replayed turn is paid for again, and a new session would restart Claude's running - # totals under the baseline the difference is taken from. - session.STAGE_IDLE_RETRIES = 0 backend = session.resolve() agent_source = resolve_source( "repo", @@ -494,11 +251,11 @@ async def _reading( ) if not finished: - await _stop(stage, task, stopping) + await _stop(stage, task, stopping, wait) # A request cut off by the stop may be billed without being in the last total. saved = _saved(output) if saved == FAILURE_FILE: - return _EXIT_REFUSED, False + return EXIT_REFUSED, False if saved == CONTRACT_FILE: return _document(job, source, output), False return _timed_out(output, soft_limit), False @@ -511,7 +268,7 @@ async def _reading( # The quiet turn's spend never arrived; what was saved before it still stands. saved = _saved(output) if saved == FAILURE_FILE: - return _EXIT_REFUSED, False + return EXIT_REFUSED, False if saved == CONTRACT_FILE: return _document(job, source, output), False return ( @@ -535,14 +292,16 @@ async def _reading( return _classify(job, source, output, stage, error), complete -async def _read(job: HarnessJob, source: Path, output: Path) -> int: +async def _read(job: HarnessJob, source: Path, output: Path, *, interrupt_wait: float) -> int: # The same instant the helper's hard limit counts from, give or take a few lines. started = time.monotonic() output = Path(output) - code, complete = _EXIT_RETRYABLE, False + code, complete = EXIT_RETRYABLE, False try: try: - code, complete = await _reading(job, Path(source), output, started, _soft_limit()) + code, complete = await _reading( + job, Path(source), output, started, _soft_limit(), interrupt_wait + ) except Exception as exc: # A contract the gate saved is paid for; a crash would have the platform pay again. if _saved(output) != CONTRACT_FILE: @@ -555,12 +314,20 @@ async def _read(job: HarnessJob, source: Path, output: Path) -> int: return code -READ_AGENT = ReadingMode( - name="read_agent", - check_job=_check_job, - work=_read, - time_limit_seconds=_soft_limit() + _HARD_LIMIT_MARGIN_SECONDS, - timed_out_code=AGENT_READING_TIMED_OUT, - timed_out_retryable=False, - crashed_code=AGENT_READING_CRASHED, -) +def _read_agent(*, interrupt_wait: float = _INTERRUPT_WAIT_SECONDS) -> ReadingMode: + return ReadingMode( + name="read_agent", + check_job=_check_job, + work=functools.partial(_read, interrupt_wait=interrupt_wait), + # The helper reads the variable when the reading starts, as the work does, so its + # limit is always the reading's own plus the margin. + time_limit_seconds=_DEFAULT_SOFT_LIMIT_SECONDS, + timed_out_code=AGENT_READING_TIMED_OUT, + timed_out_retryable=False, + crashed_code=AGENT_READING_CRASHED, + limit_env=_LIMIT_ENV, + limit_margin_seconds=_HARD_LIMIT_MARGIN_SECONDS, + ) + + +READ_AGENT = _read_agent() diff --git a/src/fi/alk/harness/authoring_entrypoint.py b/src/fi/alk/harness/authoring_entrypoint.py index 07f4341e1..30b4c5620 100644 --- a/src/fi/alk/harness/authoring_entrypoint.py +++ b/src/fi/alk/harness/authoring_entrypoint.py @@ -690,9 +690,17 @@ def main(argv: list[str] | None = None, *, validate_runtime: bool = False) -> in parser.error("--certify-only requires --bundle-output") return _certify_only(args) if args.read_agent_only: + import logging + from .agent_reading import READ_AGENT from .reading_entry import run_reading_mode + # The reading's start line is how its start-up is timed, and nothing else in this process + # sets up logging, so its INFO lines would otherwise be dropped. + reading_log = logging.getLogger("fi.alk.harness.reading_entry") + reading_log.setLevel(logging.INFO) + if not logging.getLogger().handlers: + reading_log.addHandler(logging.StreamHandler()) return run_reading_mode( READ_AGENT, job_path=args.job, diff --git a/src/fi/alk/harness/cli.py b/src/fi/alk/harness/cli.py index b142fe3cb..7bca093d5 100644 --- a/src/fi/alk/harness/cli.py +++ b/src/fi/alk/harness/cli.py @@ -39,7 +39,7 @@ from .session import TEXT, Event from .sessions import Session, new_id, save as save_session from .sources import resolve, supported -from .understand import load, open_stage, opening +from .understand import SUBMIT_REMINDER, load, open_stage, opening from .world.snapshot import saved as world_saved @@ -320,10 +320,7 @@ async def _understand(args: argparse.Namespace) -> int: interactive=args.interactive, until=lambda: load(destination) is not None or (destination / FAILURE_FILE).is_file(), - nudge=( - "Nothing was saved: you finished without calling submit_contract. Call it now " - "with the contract you worked out." - ), + nudge=SUBMIT_REMINDER, ) if (destination / FAILURE_FILE).is_file(): diff --git a/src/fi/alk/harness/framework_detection.py b/src/fi/alk/harness/framework_detection.py index bec4b49fa..8c6705b0f 100644 --- a/src/fi/alk/harness/framework_detection.py +++ b/src/fi/alk/harness/framework_detection.py @@ -988,20 +988,24 @@ def _contract_workdir(source: Path, bundle_dir: Path) -> Path | None: return None +def named_framework_folder( + source: Path, bundle_dir: Path, subfolder: str | None +) -> Path | None: + """The folder the job names, else the sealed contract's own; ``None`` when neither does.""" + if subfolder: + return agent_folder(source, subfolder) + return _contract_workdir(source, bundle_dir) + + def framework_folders_for( job: Any, manifest: Any, *, source: Path, bundle_dir: Path, work_directory: Path ) -> FrameworkFolders: runtime = manifest.runtime control_service = runtime.control_service if runtime.kind == "process" else None - subfolder = job.source.subfolder - folder = source - if subfolder: - folder = agent_folder(source, subfolder) - else: - workdir = _contract_workdir(source, bundle_dir) - if workdir is not None: - folder = workdir - elif control_service: + folder = named_framework_folder(source, bundle_dir, job.source.subfolder) + if folder is None: + folder = source + if control_service: for process in manifest.processes: if process.name == control_service: folder = bundle_dir / process.working_directory diff --git a/src/fi/alk/harness/reading_entry.py b/src/fi/alk/harness/reading_entry.py index d3ae470db..40812677d 100644 --- a/src/fi/alk/harness/reading_entry.py +++ b/src/fi/alk/harness/reading_entry.py @@ -10,17 +10,22 @@ from __future__ import annotations import asyncio +import functools import json +import logging +import math import os -import sys -import tempfile import threading from collections.abc import Awaitable, Callable from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path +from typing import Any + +from pydantic import ValidationError from . import spend +from .agent_type_check import EXIT_REFUSED from .failure_file import write_failure_file from .job import FailureDomain, HarnessFailure, HarnessJob, HarnessStage from .outbound import redact_outbound_text @@ -30,8 +35,8 @@ MESSAGE_LIMIT = 2000 COST_FILE = "cost.json" -_EXIT_RETRYABLE = 1 -_EXIT_REFUSED = 78 +EXIT_DONE = 0 +EXIT_RETRYABLE = 1 # Neither file is deleted by the hosted authoring entrypoint, so a read here takes nothing away # from the code that loads them. @@ -39,7 +44,10 @@ Path("/run/futureagi/secrets.json"), Path("/run/futureagi/simulator-secrets.json"), ) -_secret_values: tuple[str, ...] | None = None + +logger = logging.getLogger(__name__) +# Unmasked on purpose: the mask reads the secrets, and the trace of failing to read them holds none. +_secrets_logger = logging.getLogger(f"{__name__}.secrets") @dataclass(frozen=True) @@ -51,63 +59,98 @@ class ReadingMode: timed_out_code: str timed_out_retryable: bool crashed_code: str + # When set, the limit is read from this variable as the reading starts, with + # ``time_limit_seconds`` as its default; the helper's own limit adds the margin to it. + limit_env: str | None = None + limit_margin_seconds: float = 0.0 + + +def limit_seconds(env: str, default: float) -> float: + """The seconds ``env`` names when they are a positive, finite number, else ``default``.""" + try: + seconds = float(os.environ.get(env, "")) + except ValueError: + return default + return seconds if math.isfinite(seconds) and seconds > 0 else default + + +def _hard_limit(mode: ReadingMode) -> float | None: + if mode.time_limit_seconds is None: + return None + base = ( + limit_seconds(mode.limit_env, mode.time_limit_seconds) + if mode.limit_env + else mode.time_limit_seconds + ) + return base + mode.limit_margin_seconds -def _read_secret_values() -> tuple[str, ...]: +@functools.cache +def _read_secret_values(paths: tuple[Path, ...]) -> tuple[str, ...]: # Imported here: the module is large and the find-agents mode shares this helper. A broken # import must not end a reading before its failure can be written, so it masks nothing extra. try: from .hosted_entrypoint import peek_secret_values - return tuple(value for path in _SECRET_FILES for value in peek_secret_values(path)) + return tuple(value for path in paths for value in peek_secret_values(path)) except Exception: + _secrets_logger.warning("secret values not read; only URL userinfo is masked", exc_info=True) return () +def _masked(text: str) -> str: + return redact_outbound_text(text, _read_secret_values(_SECRET_FILES)) + + +class MaskSecrets(logging.Filter): + """Masks a logger's lines, stack traces included, the way a reading's failure is masked.""" + + def filter(self, record: logging.LogRecord) -> bool: + record.msg, record.args = _masked(record.getMessage()), None + if record.exc_info: + # Kept only as masked text, so no handler can format the raw exception itself. + record.exc_text = _masked(logging.Formatter().formatException(record.exc_info)) + record.exc_info = None + return True + + +logger.addFilter(MaskSecrets()) + + def write_reading_failure(failure: HarnessFailure, output: Path) -> Path | None: """Leave ``failure`` for the platform with its message made safe to show; the first one stands. Details are written as given, so a mode never puts what a provider said in them. """ try: - secrets = _secret_values if _secret_values is not None else _read_secret_values() # Masked before the cut, so a secret split by the cut can't survive half-matched. - message = redact_outbound_text(failure.message, secrets)[:MESSAGE_LIMIT] + message = _masked(failure.message)[:MESSAGE_LIMIT] return write_failure_file(failure.model_copy(update={"message": message}), Path(output)) except Exception: + logger.warning("failure file not written: code=%s", failure.code, exc_info=True) return None def write_reading_cost(output: Path, *, complete: bool) -> bool: - """Write the spend so far to ``cost.json``; ``True`` when it was written. + """Write the spend so far to ``cost.json``, and again after every turn; ``True`` when written. ``complete`` is kept only in this file, never in ``spend.snapshot()``, which builds share. """ try: - output = Path(output) - output.mkdir(parents=True, exist_ok=True) - snapshot = spend.snapshot() - # A turn with no price makes the total a floor, and a floor billed as the total is wrong. - body = {**snapshot, "complete": bool(complete) and snapshot["unpriced_turns"] == 0} - # Written whole, then moved: a poll that reads mid-write must never see half a total. - handle = tempfile.NamedTemporaryFile( - "w", dir=output, prefix=".cost-", suffix=".tmp", encoding="utf-8", delete=False - ) - try: - with handle as writing: - json.dump(body, writing, indent=2, sort_keys=True) - os.replace(handle.name, output / COST_FILE) - except Exception: - Path(handle.name).unlink(missing_ok=True) - raise + return spend.journal_to(Path(output) / COST_FILE, complete=complete) except Exception: # Accounting must never be the reason a reading fails. + logger.warning("cost file not written", exc_info=True) return False - return True -def _failure( - domain: FailureDomain, code: str, message: str, *, retryable: bool +def reading_failure( + domain: FailureDomain, + code: str, + message: str, + *, + retryable: bool, + details: dict[str, Any] | None = None, ) -> HarnessFailure: return HarnessFailure( domain=domain, @@ -115,10 +158,22 @@ def _failure( code=code, message=message, retryable=retryable, + details=details or {}, ) -def _holds_anything(output: Path) -> bool: +def _unreadable_job(exc: Exception) -> str: + # Only where and how it failed: pydantic's own text repeats the job, and a job may hold a token. + if not isinstance(exc, ValidationError): + return type(exc).__name__ + found = "; ".join( + f"{'.'.join(str(part) for part in error['loc']) or 'job'}: {error['type']}" + for error in exc.errors(include_input=False) + ) + return f"ValidationError: {found}" + + +def _output_in_use(output: Path) -> bool: try: if not os.path.lexists(output): return False @@ -128,18 +183,20 @@ def _holds_anything(output: Path) -> bool: return True -def _start_hard_limit(mode: ReadingMode, output: Path) -> threading.Timer | None: - if mode.time_limit_seconds is None: +def _start_hard_limit( + mode: ReadingMode, output: Path, limit: float | None +) -> threading.Timer | None: + if limit is None: return None - exit_code = _EXIT_RETRYABLE if mode.timed_out_retryable else _EXIT_REFUSED + exit_code = EXIT_RETRYABLE if mode.timed_out_retryable else EXIT_REFUSED def end_the_process() -> None: try: write_reading_failure( - _failure( + reading_failure( FailureDomain.INFRASTRUCTURE, mode.timed_out_code, - f"The {mode.name} reading ran past its {mode.time_limit_seconds:g} s limit " + f"The {mode.name} reading ran past its {limit:g} s limit " "and was stopped.", retryable=mode.timed_out_retryable, ), @@ -150,38 +207,50 @@ def end_the_process() -> None: # Work running in a thread can't be cancelled, and a normal exit would wait for it. os._exit(exit_code) - timer = threading.Timer(mode.time_limit_seconds, end_the_process) + timer = threading.Timer(limit, end_the_process) timer.daemon = True timer.start() return timer +async def _work_then_stop_the_clock( + mode: ReadingMode, job: HarnessJob, source: Path, output: Path, timer: threading.Timer | None +) -> int: + try: + return await mode.work(job, source, output) + finally: + # Inside the loop: asyncio.run waits for leftover worker threads before it returns, and + # the limit must not fire during that wait over a reading that has already ended. + if timer is not None: + timer.cancel() + + def run_reading_mode(mode: ReadingMode, *, job_path: Path, source: Path, output: Path) -> int: """Refuse what can't be read, start the cost at zero, then run the mode's work.""" - global _secret_values - _secret_values = _read_secret_values() + _read_secret_values(_SECRET_FILES) output = Path(output) try: job = HarnessJob.model_validate(json.loads(Path(job_path).read_text(encoding="utf-8"))) - except Exception as exc: + except (OSError, ValueError, RecursionError) as exc: write_reading_failure( - _failure( + reading_failure( FailureDomain.PLATFORM_SYNC, READING_JOB_INVALID, - f"The job file could not be read as a harness job ({type(exc).__name__}: {exc}).", + f"The job file could not be read as a harness job ({_unreadable_job(exc)}).", retryable=False, ), output, ) - return _EXIT_REFUSED + return EXIT_REFUSED try: refused = mode.check_job(job) except Exception as exc: + logger.warning("the %s job check crashed", mode.name, exc_info=True) # Nothing has been spent yet, so no cost file: a broken check is a crash, not a refusal. write_reading_failure( - _failure( + reading_failure( FailureDomain.INFRASTRUCTURE, mode.crashed_code, f"{type(exc).__name__}: {exc}", @@ -189,16 +258,16 @@ def run_reading_mode(mode: ReadingMode, *, job_path: Path, source: Path, output: ), output, ) - return _EXIT_RETRYABLE + return EXIT_RETRYABLE if refused is not None: write_reading_failure(refused, output) - return _EXIT_REFUSED + return EXIT_REFUSED # After the job checks on purpose: a stale folder is refused either way, and the failure # they wrote into it is no less true. - if _holds_anything(output): + if _output_in_use(output): write_reading_failure( - _failure( + reading_failure( FailureDomain.PLATFORM_SYNC, READING_OUTPUT_NOT_EMPTY, "The reading's output folder already holds files; give it an empty folder.", @@ -206,12 +275,12 @@ def run_reading_mode(mode: ReadingMode, *, job_path: Path, source: Path, output: ), output, ) - return _EXIT_REFUSED + return EXIT_REFUSED if not write_reading_cost(output, complete=False): # A model paid for while its cost can't be recorded would read as nothing spent. write_reading_failure( - _failure( + reading_failure( FailureDomain.INFRASTRUCTURE, mode.crashed_code, "The reading could not write its cost file, so it stopped before any model ran.", @@ -219,18 +288,16 @@ def run_reading_mode(mode: ReadingMode, *, job_path: Path, source: Path, output: ), output, ) - return _EXIT_RETRYABLE + return EXIT_RETRYABLE - timer = _start_hard_limit(mode, output) + # Read now, not when the mode was made, so it is the same limit the work reads as it starts. + timer = _start_hard_limit(mode, output, _hard_limit(mode)) crash: Exception | None = None try: - print( - f"reading started at {datetime.now(timezone.utc).isoformat()}", - file=sys.stderr, - flush=True, - ) - code = asyncio.run(mode.work(job, source, output)) + logger.info("reading started at %s", datetime.now(timezone.utc).isoformat()) + code = asyncio.run(_work_then_stop_the_clock(mode, job, source, output, timer)) except Exception as exc: + logger.error("the %s reading crashed", mode.name, exc_info=True) crash = exc finally: # Cancelled first, so the limit can't fire during a crash's writes or the caller's cleanup. @@ -239,7 +306,7 @@ def run_reading_mode(mode: ReadingMode, *, job_path: Path, source: Path, output: if crash is not None: write_reading_failure( - _failure( + reading_failure( FailureDomain.INFRASTRUCTURE, mode.crashed_code, f"{type(crash).__name__}: {crash}", @@ -248,6 +315,6 @@ def run_reading_mode(mode: ReadingMode, *, job_path: Path, source: Path, output: output, ) write_reading_cost(output, complete=False) - return _EXIT_RETRYABLE + return EXIT_RETRYABLE # The work wrote its own final cost; rewriting it here could mark a lost turn as paid. return code diff --git a/src/fi/alk/harness/reading_result.py b/src/fi/alk/harness/reading_result.py new file mode 100644 index 000000000..a4940d058 --- /dev/null +++ b/src/fi/alk/harness/reading_result.py @@ -0,0 +1,250 @@ +"""What a finished reading leaves: the agent's description from its saved contract, or why not. + +A saved contract has been paid for, so once one exists nothing after it is retried. Without one, +the reading's own turns decide whether a retry could get further. +""" + +from __future__ import annotations + +import json +import logging +import os +import re +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from .. import __version__ +from ..providers.agent import AgentBuildError, build_authored_agent +from .agent_type_check import EXIT_REFUSED +from .backends import StageDone +from .contract import AgentContract +from .failure_file import FAILURE_FILE +from .framework_detection import detect_framework, named_framework_folder +from .job import FailureDomain, HarnessJob +from .reading_entry import EXIT_DONE, EXIT_RETRYABLE, reading_failure, write_reading_failure + +if TYPE_CHECKING: + from .agent_reading import _ReadingStage + +logger = logging.getLogger(__name__) + +AGENT_FILE = "agent.json" +CONTRACT_FILE = "contract.json" + +AGENT_READING_MODEL_ERROR = "agent_reading_model_error" +AGENT_READING_INCOMPLETE = "agent_reading_incomplete" +AGENT_READING_TOO_LARGE = "agent_reading_too_large" +AGENT_DOCUMENT_INVALID = "agent_document_invalid" + +# The platform's own caps, compared the way it compares them. +_DOCUMENT_LIMIT_BYTES = 1_048_576 +_CONTRACT_LIMIT_BYTES = 5 * 1_048_576 + +# A model out of turns can't save, so a reminder after one only adds cost. +_TURN_LIMITS = frozenset({"max_turns", "error_max_turns"}) +# Matched only, never repeated in a message: these are ADK's words, not ours. +_CALL_LIMIT = re.compile(r"Max number of llm calls limit of `\d+` exceeded") +# Statuses a retry may get past; any other 4xx would only be refused again. +_PASSING_STATUSES = frozenset({408, 409, 429}) + + +def _saved(output: Path) -> str: + if (output / FAILURE_FILE).is_file(): + return FAILURE_FILE + if (output / CONTRACT_FILE).is_file(): + return CONTRACT_FILE + return "" + + +def _out_of_turns(done: StageDone) -> bool: + # ADK enforces its call budget by raising, which the Gemini backend reports as a failure with + # no status, so ADK's own words are the signal. Matched whole, since the phrase inside other + # text may be the model's own. The count is no signal: it catches no real limit, and it would + # read a request cut on the last allowed call as one. + return done.outcome in _TURN_LIMITS or ( + done.is_error + and done.api_error_status is None + and any(_CALL_LIMIT.fullmatch(str(said)) for said in done.errors) + ) + + +def _refusing(status: Any) -> bool: + return ( + isinstance(status, int) + and not isinstance(status, bool) + and 400 <= status <= 499 + and status not in _PASSING_STATUSES + ) + + +def _model_error(output: Path, message: str, status: Any = None, *, retryable: bool) -> int: + write_reading_failure( + reading_failure( + FailureDomain.INFRASTRUCTURE, + AGENT_READING_MODEL_ERROR, + message, + retryable=retryable, + # Only our own reading of the status: what the provider said stays in the message. + details={"status": status if isinstance(status, int) else None}, + ), + output, + ) + return EXIT_RETRYABLE if retryable else EXIT_REFUSED + + +def _classify( + job: HarnessJob, + source: Path, + output: Path, + stage: _ReadingStage, + error: BaseException | None, +) -> int: + saved = _saved(output) + if saved == FAILURE_FILE: + return EXIT_REFUSED + # Before any error: the gate accepted the reading, and refusing it now would pay for it twice. + if saved == CONTRACT_FILE: + return _document(job, source, output) + forbidden = stage.forbidden_models() + if forbidden: + # A retry would bill the same model again. + return _model_error( + output, + "The provider billed a model this harness may not spend on (" + + ", ".join(sorted(forbidden)) + + "), so the reading was stopped.", + retryable=False, + ) + # A last turn out of turns spent a whole budget, so a retry would likely spend another and + # stop the same way, whatever failed before it. + ran_out = bool(stage.done) and _out_of_turns(stage.done[-1]) + failed = [ + done + for done in stage.done + if done.api_error_status is not None + or (done.is_error and not _out_of_turns(done)) + ] + if not ran_out and (failed or error is not None): + status = next( + (done.api_error_status for done in reversed(failed) if done.api_error_status is not None), + None, + ) + said = [str(one) for done in failed for one in done.errors] + if error is not None: + said.append(f"{type(error).__name__}: {error}") + shown = f" (status {status})" if isinstance(status, int) else "" + return _model_error( + output, + f"The model provider failed while the agent was being read{shown}: " + + ("; ".join(said) or "no detail was given."), + status, + retryable=not _refusing(status), + ) + write_reading_failure( + reading_failure( + FailureDomain.INFRASTRUCTURE, + AGENT_READING_INCOMPLETE, + "The model finished without saving the agent's description.", + retryable=False, + details={ + "outcome": stage.done[-1].outcome if stage.done else "", + "turns": sum(int(done.turns or 0) for done in stage.done), + }, + ), + output, + ) + return EXIT_REFUSED + + +class _TooLarge(Exception): + def __init__(self, sizes: dict[str, int]) -> None: + super().__init__("too large") + self.sizes = sizes + + +def _framework_folder(job: HarnessJob, source: Path, output: Path) -> Path: + # As a build chooses it when there is no build: the named folder, else the contract's own. + return named_framework_folder(source, output, job.source.subfolder) or source + + +def _write_whole(path: Path, body: bytes) -> None: + # Written whole, then moved: the platform must never read half a document. + handle = tempfile.NamedTemporaryFile( + "wb", dir=path.parent, prefix=".agent-", suffix=".tmp", delete=False + ) + try: + with handle as writing: + writing.write(body) + os.replace(handle.name, path) + except BaseException: + Path(handle.name).unlink(missing_ok=True) + raise + + +def _document(job: HarnessJob, source: Path, output: Path) -> int: + """The agent's description from the saved contract, as a build makes it.""" + # One catch over every step: a saved contract is paid for, so nothing after it is retryable. + try: + contract_bytes = (output / CONTRACT_FILE).stat().st_size + if contract_bytes > _CONTRACT_LIMIT_BYTES: + raise _TooLarge({"contract_bytes": contract_bytes}) + contract = AgentContract.model_validate( + json.loads((output / CONTRACT_FILE).read_text(encoding="utf-8")) + ).model_dump(mode="json") + # The echo proves which folder was read, so it comes from the contract. + built_from = str((contract.get("runtime") or {}).get("source_subfolder") or "") + if built_from != (job.source.subfolder or ""): + raise AgentBuildError("subfolder_mismatch") + framework = detect_framework( + _framework_folder(job, source, output), repo_root=source, package_dir=None + ) + inner = build_authored_agent( + contract, subfolder=built_from or None, framework=framework + ) + body = json.dumps( + {"sdk_version": __version__, "agent": inner}, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + stored = json.dumps(inner, ensure_ascii=False).encode("utf-8") + if len(body) > _DOCUMENT_LIMIT_BYTES or len(stored) > _DOCUMENT_LIMIT_BYTES: + raise _TooLarge( + { + "agent_bytes": len(body), + "stored_bytes": len(stored), + "contract_bytes": contract_bytes, + } + ) + _write_whole(output / AGENT_FILE, body) + except _TooLarge as too_large: + write_reading_failure( + reading_failure( + FailureDomain.INFRASTRUCTURE, + AGENT_READING_TOO_LARGE, + "The agent's description is larger than the platform stores (1 MiB for the " + "description, 5 MiB for the contract).", + retryable=False, + details=dict(too_large.sizes), + ), + output, + ) + return EXIT_REFUSED + except Exception as exc: + code = str(getattr(exc, "code", "") or "") + logger.warning( + "agent document not built: class=%s code=%s", type(exc).__name__, code + ) + write_reading_failure( + reading_failure( + FailureDomain.INFRASTRUCTURE, + AGENT_DOCUMENT_INVALID, + "The agent's description could not be built from its saved contract " + f"({type(exc).__name__}{': ' + code if code else ''}).", + retryable=False, + details={"class": type(exc).__name__, "code": code}, + ), + output, + ) + return EXIT_REFUSED + return EXIT_DONE diff --git a/src/fi/alk/harness/session.py b/src/fi/alk/harness/session.py index 8b1ca2aaa..739f04ba1 100644 --- a/src/fi/alk/harness/session.py +++ b/src/fi/alk/harness/session.py @@ -217,9 +217,12 @@ def __init__( name: str = "", backend: HarnessBackend | None = None, overheard: bool = True, + idle_retries: int | None = None, ) -> None: self._spec = spec self._backend = backend + # None follows the process-wide setting, read when each message is sent. + self._idle_retries = idle_retries # Whether this stage's turns are echoed to the run's conversation channel. self._overheard = overheard self._session: HarnessSession | None = None @@ -454,14 +457,15 @@ async def say( self, message: str, *, on_event: Callable[[Event], None] | None = None ) -> Turn: """Send a message and wait for the whole reply.""" - for attempt in range(STAGE_IDLE_RETRIES + 1): + retries = STAGE_IDLE_RETRIES if self._idle_retries is None else self._idle_retries + for attempt in range(retries + 1): try: async for event in self.stream(message): if on_event: on_event(event) return self.history[-1] except StageIdleTimeout: - if attempt >= STAGE_IDLE_RETRIES: + if attempt >= retries: raise # A timed-out receive has been cancelled and the provider session may still be # waiting on a dead stream. Reusing it can only reproduce the dead stream. diff --git a/src/fi/alk/harness/spend.py b/src/fi/alk/harness/spend.py index 34fb36b77..a9ecd6683 100644 --- a/src/fi/alk/harness/spend.py +++ b/src/fi/alk/harness/spend.py @@ -13,6 +13,7 @@ from __future__ import annotations import json +import logging import os import tempfile from pathlib import Path @@ -20,16 +21,23 @@ JOURNAL_ALIAS = "ALK_SPEND_JOURNAL" +logger = logging.getLogger(__name__) + _stages: dict[str, dict[str, Any]] = {} _path: Path | None = None +_complete: bool | None = None + +def journal_to(path: str | os.PathLike[str] | None, *, complete: bool | None = None) -> bool: + """Where to keep the ledger. Nothing is written until this is set or the alias names a path. -def journal_to(path: str | os.PathLike[str] | None) -> None: - """Where to keep the ledger. Nothing is written until this is set or the alias names a path.""" - global _path + ``complete`` is written beside the totals only when given: a reading's bill says whether every + turn's spend is in it, and a build's ledger has no such key. ``True`` when the file was written. + """ + global _path, _complete _path = Path(path) if path else None - if _path is not None: - _flush() + _complete = complete + return _flush() if _path is not None else False def _destination() -> Path | None: @@ -126,10 +134,15 @@ def snapshot() -> dict[str, Any]: } -def _flush() -> None: +def _flush() -> bool: destination = _destination() if destination is None: - return + return False + body = snapshot() + if _complete is not None: + # A turn with no price makes the total a floor, and a floor billed as the total is wrong. + body["complete"] = _complete and body["unpriced_turns"] == 0 + handle = None try: destination.parent.mkdir(parents=True, exist_ok=True) # Written whole, then moved: a poll that reads mid-write must never see half a total. @@ -137,8 +150,16 @@ def _flush() -> None: "w", dir=destination.parent, prefix=".spend-", suffix=".json", delete=False ) with handle as writing: - json.dump(snapshot(), writing, indent=2, sort_keys=True) + json.dump(body, writing, indent=2, sort_keys=True) os.replace(handle.name, destination) except OSError: # Accounting must never be the reason a stage fails. - pass + logger.warning("spend file not written: %s", destination, exc_info=True) + if handle is not None: + # Left behind, it would sit in a folder the platform reads as the run's output. + try: + Path(handle.name).unlink(missing_ok=True) + except OSError: + pass + return False + return True diff --git a/src/fi/alk/harness/understand.py b/src/fi/alk/harness/understand.py index 0d7d76371..976a6c4bb 100644 --- a/src/fi/alk/harness/understand.py +++ b/src/fi/alk/harness/understand.py @@ -40,6 +40,11 @@ "the contract." ) PROVIDER_IMPORT_PROFILE_PATH_ENV = "ALK_PROVIDER_IMPORT_PROFILE_PATH" +# Said once when a model ends its turn without saving; every unattended reading uses these words. +SUBMIT_REMINDER = ( + "Nothing was saved: you finished without calling submit_contract. Call it now " + "with the contract you worked out." +) def _provider_import_briefing() -> str: diff --git a/tests/harness/test_agent_reading.py b/tests/harness/test_agent_reading.py index b649773d8..f68fef632 100644 --- a/tests/harness/test_agent_reading.py +++ b/tests/harness/test_agent_reading.py @@ -6,13 +6,11 @@ from __future__ import annotations -import ast import asyncio -import inspect import json +import logging import os import shutil -import textwrap import threading import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -21,7 +19,14 @@ import pytest import fi.alk -from fi.alk.harness import agent_reading, authoring_entrypoint, reading_entry, session, spend +from fi.alk.harness import ( + agent_reading, + authoring_entrypoint, + reading_entry, + reading_result, + session, + spend, +) from fi.alk.harness.backends import StageDone from fi.alk.harness.builder import BUILDER_TURNS from fi.alk.harness.config import chosen_model @@ -30,6 +35,7 @@ from fi.alk.harness.job import HarnessJob, SourceKind from fi.alk.harness.reading_entry import READING_JOB_INVALID, run_reading_mode from fi.alk.harness.tools import CONTRACT_SERVER +from fi.alk.harness.understand import SUBMIT_REMINDER from fi.alk.providers.agent import build_authored_agent EXAMPLES = Path(__file__).resolve().parents[2] / "examples" / "frameworks" @@ -47,22 +53,17 @@ **CHAT, "runtime": {"language": "python", "workdir": SUPPORT, "source_subfolder": SUPPORT}, } -# The real one, kept before the fixture shortens the limit for every other test. -_SOFT_LIMIT = agent_reading._soft_limit +_LIMIT_ENV = "ALK_READING_MAX_DURATION_SECONDS" @pytest.fixture(autouse=True) def _fresh_state(monkeypatch, tmp_path): - monkeypatch.setattr(session, "STAGE_IDLE_RETRIES", session.STAGE_IDLE_RETRIES) monkeypatch.delenv(spend.JOURNAL_ALIAS, raising=False) - monkeypatch.delenv("ALK_READING_MAX_DURATION_SECONDS", raising=False) monkeypatch.setenv("ALK_HARNESS_MODEL", "gemini-2.5-flash") # The machine's own secrets files must not decide what a test sees masked. monkeypatch.setattr(reading_entry, "_SECRET_FILES", (tmp_path / "no-secrets.json",)) - monkeypatch.setattr(reading_entry, "_secret_values", None) # A scripted model that never answers must end a test, not hang it. - monkeypatch.setattr(agent_reading, "_soft_limit", lambda: 20.0) - monkeypatch.setattr(agent_reading, "_INTERRUPT_WAIT_SECONDS", 1.0) + monkeypatch.setenv(_LIMIT_ENV, "20") spend._stages.clear() spend.journal_to(None) yield @@ -82,6 +83,8 @@ def __init__(self, backend: "_Backend", spec) -> None: self.interrupted = asyncio.Event() async def start(self) -> None: + if self.backend.saves_on_start is not None: + _save(self.backend.output, self.backend.saves_on_start) if self.backend.start_raises: raise FileNotFoundError("the Claude CLI was not found") @@ -123,6 +126,7 @@ def __init__(self, name: str, output: Path, turns: list) -> None: self.seen: list = [] self.interrupt_raises = False self.start_raises = False + self.saves_on_start: dict | None = None self.submissions = 0 def create(self, spec) -> _Live: @@ -233,11 +237,18 @@ def _repo(root: Path) -> Path: return source -def _run(root: Path, job: dict | None = None, *, source: Path | None = None, output=None) -> int: +def _run( + root: Path, + job: dict | None = None, + *, + source: Path | None = None, + output=None, + wait: float = 1.0, +) -> int: job_path = root / "job.json" job_path.write_text(json.dumps(job or _job()), encoding="utf-8") return run_reading_mode( - agent_reading.READ_AGENT, + agent_reading._read_agent(interrupt_wait=wait), job_path=job_path, source=source if source is not None else root / "source", output=output if output is not None else root / "reading", @@ -273,9 +284,8 @@ def opened(*args, **kwargs): monkeypatch.setattr(agent_reading, "open_stage", opened) -def _short_limit(monkeypatch, soft: float = 0.3, wait: float = 0.3) -> None: - monkeypatch.setattr(agent_reading, "_soft_limit", lambda: soft) - monkeypatch.setattr(agent_reading, "_INTERRUPT_WAIT_SECONDS", wait) +def _short_limit(monkeypatch, soft: float = 0.3) -> None: + monkeypatch.setenv(_LIMIT_ENV, str(soft)) # -- R1 to R9: the files a reading leaves ------------------------------------------------------- @@ -511,7 +521,7 @@ def test_an_oversized_document_is_refused(tmp_path, model, monkeypatch, inner, p document = inner() if not passes and document["items"]: assert _body_size(document) <= MIB - monkeypatch.setattr(agent_reading, "build_authored_agent", lambda *a, **k: document) + monkeypatch.setattr(reading_result, "build_authored_agent", lambda *a, **k: document) model(_writes(CHAT, cost_usd=0.05)) output = tmp_path / "reading" @@ -562,7 +572,7 @@ def test_no_contract_after_the_reminder_is_incomplete(tmp_path, model): assert _run(tmp_path) == 78 assert len(fake.sent) == 2 - assert fake.sent[1] == agent_reading._REMINDER + assert fake.sent[1] == SUBMIT_REMINDER failure = _failure_in(output) assert (failure["code"], failure["retryable"]) == ("agent_reading_incomplete", False) assert failure["details"] == {"outcome": "success", "turns": 2} @@ -690,7 +700,7 @@ def test_an_error_flag_without_a_status_still_gets_the_reminder(tmp_path, model) assert _run(tmp_path) == 0 - assert fake.sent[1:] == [agent_reading._REMINDER] + assert fake.sent[1:] == [SUBMIT_REMINDER] assert (output / "agent.json").exists() # The first turn's error had no status, so its spend may not all be in the total. _assert_cost(output, 0.2, False) @@ -842,7 +852,7 @@ def test_a_cut_on_the_last_allowed_call_is_not_a_turn_limit(tmp_path, model): assert _run(tmp_path) == 1 - assert fake.sent[1:] == [agent_reading._REMINDER] + assert fake.sent[1:] == [SUBMIT_REMINDER] failure = _failure_in(output) assert (failure["code"], failure["retryable"]) == ("agent_reading_model_error", True) # The cut request may be billed without its total reaching the stage. @@ -934,6 +944,8 @@ def test_an_idle_stream_is_not_replayed(tmp_path, model, monkeypatch): assert len(fake.created) == 1 assert fake.interrupts == [] + # Only this stage skips the replay; the process's own setting is left as it was. + assert session.STAGE_IDLE_RETRIES == 1 failure = _failure_in(output) assert (failure["code"], failure["retryable"]) == ("agent_reading_model_error", True) _assert_cost(output, 0.0, False) @@ -960,7 +972,7 @@ async def saves_then_goes_quiet(live): def test_the_soft_limit_interrupts_once_and_charges_what_arrives(tmp_path, model, monkeypatch): _repo(tmp_path) - _short_limit(monkeypatch, wait=2.0) + _short_limit(monkeypatch) async def answers_the_interrupt(live): await live.interrupted.wait() @@ -969,7 +981,7 @@ async def answers_the_interrupt(live): fake = model(answers_the_interrupt) output = tmp_path / "reading" - assert _run(tmp_path) == 78 + assert _run(tmp_path, wait=2.0) == 78 assert len(fake.sent) == 1 assert len(fake.interrupts) == 1 @@ -983,7 +995,7 @@ def test_the_soft_limit_marks_cost_incomplete_when_nothing_arrives(tmp_path, mod fake = model(_quiet) output = tmp_path / "reading" - assert _run(tmp_path) == 78 + assert _run(tmp_path, wait=0.3) == 78 assert len(fake.interrupts) == 1 failure = _failure_in(output) @@ -991,21 +1003,54 @@ def test_the_soft_limit_marks_cost_incomplete_when_nothing_arrives(tmp_path, mod _assert_cost(output, 0.0, False) -def test_an_interrupt_that_raises_is_still_a_timeout(tmp_path, model, monkeypatch): +def test_an_interrupt_that_raises_is_still_a_timeout(tmp_path, model, monkeypatch, caplog): _repo(tmp_path) _short_limit(monkeypatch) fake = model(_quiet) fake.interrupt_raises = True output = tmp_path / "reading" - assert _run(tmp_path) == 78 + with caplog.at_level("WARNING", logger=agent_reading.__name__): + assert _run(tmp_path, wait=0.3) == 78 + + assert _failure_in(output)["code"] == "agent_reading_timed_out" + [logged] = [ + record + for record in caplog.records + if record.name == agent_reading.__name__ and "not delivered" in record.getMessage() + ] + assert "RuntimeError: the interrupt control request timed out" in logged.exc_text + + +def test_a_session_that_breaks_after_the_stop_is_logged_and_still_a_timeout( + tmp_path, model, monkeypatch, caplog +): + _repo(tmp_path) + _short_limit(monkeypatch) + + async def breaks_when_interrupted(live): + await live.interrupted.wait() + raise ConnectionResetError("the stream broke after the stop") + yield _done() + + model(breaks_when_interrupted) + output = tmp_path / "reading" + + with caplog.at_level("WARNING", logger=agent_reading.__name__): + assert _run(tmp_path, wait=1.0) == 78 assert _failure_in(output)["code"] == "agent_reading_timed_out" + [logged] = [ + record + for record in caplog.records + if record.name == agent_reading.__name__ and "ended with an error" in record.getMessage() + ] + assert "ConnectionResetError: the stream broke after the stop" in logged.exc_text def test_a_contract_saved_at_the_soft_limit_is_kept(tmp_path, model, monkeypatch): _repo(tmp_path) - _short_limit(monkeypatch, wait=2.0) + _short_limit(monkeypatch) async def saves_when_interrupted(live): await live.interrupted.wait() @@ -1015,7 +1060,7 @@ async def saves_when_interrupted(live): model(saves_when_interrupted) output = tmp_path / "reading" - assert _run(tmp_path) == 0 + assert _run(tmp_path, wait=2.0) == 0 assert (output / "agent.json").exists() assert not (output / "failure.json").exists() @@ -1024,7 +1069,7 @@ async def saves_when_interrupted(live): def test_nothing_is_saved_after_the_soft_limit_decides(tmp_path, model, monkeypatch): _repo(tmp_path) - _short_limit(monkeypatch, soft=0.3, wait=0.3) + _short_limit(monkeypatch, soft=0.3) async def saves_too_late(live): # Ignores the interrupt, and would save only after the wait has run out. @@ -1035,7 +1080,7 @@ async def saves_too_late(live): fake = model(saves_too_late) output = tmp_path / "reading" - assert _run(tmp_path) == 78 + assert _run(tmp_path, wait=0.3) == 78 # The session was closed before either result file could be written. assert fake.stopped == [{"failure": False, "agent": False}] @@ -1350,24 +1395,9 @@ def test_the_claude_backend_accepts_the_reading_spec(tmp_path, model, monkeypatc Path(prompt["path"]).unlink(missing_ok=True) -def test_the_reminder_is_the_clis_reminder(): - from fi.alk.harness import cli - - tree = ast.parse(textwrap.dedent(inspect.getsource(cli._understand))) - [nudge] = [ - keyword.value - for node in ast.walk(tree) - if isinstance(node, ast.Call) - for keyword in node.keywords - if keyword.arg == "nudge" - ] - assert isinstance(nudge, ast.Constant) - assert nudge.value == agent_reading._REMINDER - - def test_the_soft_limit_counts_from_the_readings_start(tmp_path, model, monkeypatch): _repo(tmp_path) - _short_limit(monkeypatch, soft=3.0, wait=0.2) + _short_limit(monkeypatch, soft=3.0) real = agent_reading.open_stage def slow(*args, **kwargs): @@ -1378,7 +1408,7 @@ def slow(*args, **kwargs): fake = model(_quiet) started = time.monotonic() - assert _run(tmp_path) == 78 + assert _run(tmp_path, wait=0.2) == 78 [interrupted] = fake.interrupts assert 2.8 <= interrupted - started < 4.0 @@ -1408,6 +1438,25 @@ def test_the_switch_runs_only_the_reading(tmp_path, model, monkeypatch): assert (output / "agent.json").exists() +def test_the_switch_sends_the_readings_start_to_stderr(tmp_path, model, monkeypatch, capfd): + _repo(tmp_path) + # As in the sandbox: nothing has set up logging before the switch runs. + monkeypatch.setattr(logging.getLogger(), "handlers", []) + monkeypatch.setattr(reading_entry.logger, "handlers", []) + monkeypatch.setattr(reading_entry.logger, "level", logging.NOTSET) + model(_writes(CHAT)) + job_path = tmp_path / "job.json" + job_path.write_text(json.dumps(_job()), encoding="utf-8") + output = tmp_path / "reading" + + code = authoring_entrypoint.main( + [str(job_path), "--source", str(tmp_path / "source"), "--output", str(output), "--read-agent-only"] + ) + + assert code == 0 + assert "reading started at " in capfd.readouterr().err + + def test_a_bad_job_through_main_is_refused(tmp_path, model): fake = model() job_path = tmp_path / "job.json" @@ -1459,47 +1508,40 @@ def test_nothing_is_written_outside_the_output(tmp_path, model, ends): @pytest.mark.parametrize("value", ["abc", "0", "-5", "", "nan", "inf"]) def test_a_bad_time_limit_falls_back_to_the_default(monkeypatch, value): - monkeypatch.setenv("ALK_READING_MAX_DURATION_SECONDS", value) - - assert _SOFT_LIMIT() == 1800.0 + monkeypatch.setenv(_LIMIT_ENV, value) - -def test_the_hard_limit_is_a_timeout_that_is_not_retried(): - mode = agent_reading.READ_AGENT - - assert mode.timed_out_code == agent_reading.AGENT_READING_TIMED_OUT == "agent_reading_timed_out" - assert mode.timed_out_retryable is False - assert mode.crashed_code == agent_reading.AGENT_READING_CRASHED == "agent_reading_crashed" + assert agent_reading._soft_limit() == 1800.0 + assert reading_entry._hard_limit(agent_reading.READ_AGENT) == 1800.0 + 180.0 def test_a_good_time_limit_is_read(monkeypatch): - monkeypatch.setenv("ALK_READING_MAX_DURATION_SECONDS", "120") + monkeypatch.setenv(_LIMIT_ENV, "120") - assert _SOFT_LIMIT() == 120.0 - assert agent_reading.READ_AGENT.time_limit_seconds == 1800.0 + 180.0 + assert agent_reading._soft_limit() == 120.0 + mode = agent_reading.READ_AGENT + # The helper's limit is the reading's own plus its margin, and a timeout repeats if retried. + assert reading_entry._hard_limit(mode) == 120.0 + 180.0 + assert mode.timed_out_retryable is False -def test_a_fault_after_a_saved_contract_still_leaves_the_document( - tmp_path, model, monkeypatch, caplog -): +def test_a_fault_after_a_saved_contract_still_leaves_the_document(tmp_path, model, caplog): _repo(tmp_path) - - def unreadable(*_args, **_kwargs): - raise PermissionError("the output folder could not be read") - - monkeypatch.setattr(agent_reading, "_classify", unreadable) - model(_writes(CHAT, cost_usd=0.1)) + fake = model() + # The contract lands, then the session fails to open: the fault the reading raises. + fake.saves_on_start = CHAT + fake.start_raises = True output = tmp_path / "reading" with caplog.at_level("WARNING", logger=agent_reading.__name__): assert _run(tmp_path) == 0 assert _names(output) == {"contract.json", "agent.json", "cost.json"} - _assert_cost(output, 0.1, False) + assert fake.sent == [] + _assert_cost(output, 0.0, False) # The fault is named by its class only: its text may carry what the customer's code said. logged = [record.getMessage() for record in caplog.records if record.name == agent_reading.__name__] - assert any("PermissionError" in line for line in logged) - assert not any("could not be read" in line for line in logged) + assert any("FileNotFoundError" in line for line in logged) + assert not any("was not found" in line for line in logged) def test_a_session_that_cannot_open_is_a_crash(tmp_path, model): diff --git a/tests/harness/test_reading_entry.py b/tests/harness/test_reading_entry.py index 484e443fe..f88965fd1 100644 --- a/tests/harness/test_reading_entry.py +++ b/tests/harness/test_reading_entry.py @@ -11,7 +11,7 @@ import pytest -from fi.alk.harness import reading_entry, session, spend +from fi.alk.harness import reading_entry, spend from fi.alk.harness.job import FailureDomain, HarnessFailure, HarnessJob, HarnessStage from fi.alk.harness.reading_entry import ( MESSAGE_LIMIT, @@ -43,11 +43,9 @@ @pytest.fixture(autouse=True) def _fresh_state(monkeypatch, tmp_path): - monkeypatch.setattr(session, "STAGE_IDLE_RETRIES", session.STAGE_IDLE_RETRIES) monkeypatch.delenv(spend.JOURNAL_ALIAS, raising=False) # The machine's own secrets files must not decide what a test sees masked. monkeypatch.setattr(reading_entry, "_SECRET_FILES", (tmp_path / "no-secrets.json",)) - monkeypatch.setattr(reading_entry, "_secret_values", None) spend._stages.clear() spend.journal_to(None) yield @@ -134,9 +132,10 @@ def _refusal(code: str = "scan_job_invalid") -> HarnessFailure: # The helper ends the process with os._exit at its limit, so those tests run it in a child. _CHILD = r""" -import asyncio, json, sys, time +import asyncio, json, logging, sys, time config = json.loads(sys.argv[1]) sys.path[:0] = config["sys_path"] +logging.basicConfig(level=logging.INFO) from pathlib import Path from fi.alk.harness import reading_entry, spend from fi.alk.harness.reading_entry import ReadingMode, run_reading_mode @@ -152,6 +151,14 @@ def broken(*_args, **_kwargs): async def work(job, source, output): if config["work"] == "returns": return 0 + if config["work"] == "leaves_a_thread": + (output / "agent.json").write_text('{"agent": "kept"}') + try: + # A scan given up on by its caller: the thread itself runs on past the limit. + await asyncio.wait_for(asyncio.to_thread(time.sleep, 3.0), 0.1) + except TimeoutError: + pass + return 0 if config["work"] == "records_then_blocks": spend.record("read-agent", 0.21, turns=1, models={"m"}) if config["work"] == "breaks_the_writers_then_blocks": @@ -235,6 +242,36 @@ def test_an_unreadable_job_is_refused_before_any_work(tmp_path, values, case): assert work.calls == [] +def test_a_refused_job_leaves_none_of_its_values_anywhere(tmp_path, values, capfd, caplog): + token = "ghp_REVIEWSECRET1234567890abcdef" + job_path = tmp_path / "job.json" + job_path.write_text(json.dumps({**_job(), "metadata": {"token": token}}), encoding="utf-8") + + with caplog.at_level("DEBUG"): + assert _run(_mode(values, _Work()), tmp_path, job_path) == 78 + + failure = (tmp_path / "reading" / "failure.json").read_text(encoding="utf-8") + assert json.loads(failure)["code"] == READING_JOB_INVALID + captured = capfd.readouterr() + kept = failure + captured.out + captured.err + caplog.text + assert not [token[at : at + 8] for at in range(len(token) - 7) if token[at : at + 8] in kept] + + +def test_a_job_nested_past_the_parsers_depth_is_refused(tmp_path, values): + job_path = tmp_path / "job.json" + job_path.write_text("[" * 100_000 + "]" * 100_000, encoding="utf-8") + work = _Work() + + assert _run(_mode(values, work), tmp_path, job_path) == 78 + + failure = _failure_in(tmp_path / "reading") + assert (failure["code"], failure["message"]) == ( + READING_JOB_INVALID, + "The job file could not be read as a harness job (RecursionError).", + ) + assert work.calls == [] + + def test_the_modes_own_job_check_refuses_before_any_work(tmp_path, values): seen: list = [] @@ -386,6 +423,38 @@ def test_the_time_limit_does_not_fire_after_the_work_returns(tmp_path): assert not (tmp_path / "reading" / "failure.json").exists() +def test_a_limit_named_by_the_environment_is_read_when_the_reading_starts( + tmp_path, monkeypatch +): + monkeypatch.setenv("ALK_TEST_READING_LIMIT", "0.5") + mode_values = { + **READ_AGENT_VALUES, + "time_limit_seconds": 30.0, + "limit_env": "ALK_TEST_READING_LIMIT", + "limit_margin_seconds": 0.5, + } + + finished, elapsed = _run_in_a_child(tmp_path, mode_values, "blocks") + + assert finished.returncode == 78, finished.stderr + assert 1.0 <= elapsed <= 1.0 + 5 + failure = _failure_in(tmp_path / "reading") + assert failure["code"] == "agent_reading_timed_out" + assert "past its 1 s limit" in failure["message"] + + +def test_a_thread_left_after_the_work_does_not_time_the_reading_out(tmp_path): + mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} + + finished, elapsed = _run_in_a_child(tmp_path, mode_values, "leaves_a_thread") + + assert finished.returncode == 0, finished.stderr + # The process did wait for the thread, well past the limit, and the limit stayed quiet. + assert elapsed >= 3.0 + assert not (tmp_path / "reading" / "failure.json").exists() + assert (tmp_path / "reading" / "agent.json").read_text() == '{"agent": "kept"}' + + def test_a_crash_writes_its_code_with_no_secret_and_a_capped_message(tmp_path, values): def crash(*_args): raise RuntimeError("could not fetch https://user:pw@host/repo.git " + "x" * 3000) @@ -444,19 +513,33 @@ def test_the_works_exit_code_is_returned(tmp_path, values, exit_code): def test_a_works_own_incomplete_cost_is_kept(tmp_path, values): + written: list[int] = [] + def account(_job, _source, output): spend.record("read-agent", 0.42, turns=1, models={"m"}) assert write_reading_cost(output, complete=False) - # Spend the work's file leaves out: any rewrite after the work returns would add it. - spend.record("read-agent", 0.08, turns=1, models={"m"}) + # Every write moves a new file into place, so a rewrite after the work would show. + written.append((output / "cost.json").stat().st_ino) assert _run(_mode(values, _Work(action=account)), tmp_path) == 0 - cost = _cost_in(tmp_path / "reading") + output = tmp_path / "reading" + assert (output / "cost.json").stat().st_ino == written[0] + cost = _cost_in(output) assert cost["total_usd"] == 0.42 assert cost["complete"] is False +def test_each_turn_after_the_first_write_lands_in_the_cost_file(tmp_path): + output = tmp_path / "reading" + assert write_reading_cost(output, complete=False) is True + + spend.record("read-agent", 0.25, turns=1, models={"m"}) + + assert _cost_in(output) == {**spend.snapshot(), "complete": False} + assert sorted(entry.name for entry in output.iterdir()) == ["cost.json"] + + def test_cost_is_left_after_a_timeout_and_a_crash(tmp_path): def record_then_crash(*_args): spend.record("read-agent", 0.21, turns=1, models={"m"}) @@ -501,6 +584,46 @@ def test_the_checks_run_in_order(tmp_path, values): assert work.calls == [] +def test_a_crash_keeps_its_trace_in_the_log_masked_and_out_of_the_file( + tmp_path, monkeypatch, caplog +): + secret = "sk-0123456789abcdefg" + secrets = tmp_path / "secrets.json" + secrets.write_text(json.dumps({"TARGET_KEY": secret}), encoding="utf-8") + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (secrets,)) + + def crash(*_args): + raise RuntimeError(f"the provider refused {secret}") + + with caplog.at_level("INFO", logger=reading_entry.__name__): + assert _run(_mode(READ_AGENT_VALUES, _Work(action=crash)), tmp_path) == 1 + + assert "Traceback" not in _failure_in(tmp_path / "reading")["message"] + assert "reading started at " in caplog.text + assert "Traceback (most recent call last)" in caplog.text + assert "RuntimeError: the provider refused ***" in caplog.text + assert secret not in caplog.text + + +def test_each_swallowed_error_is_logged_with_its_trace(tmp_path, monkeypatch, caplog): + blocked = tmp_path / "blocked" + blocked.write_text("a file, not a folder", encoding="utf-8") + + with caplog.at_level("WARNING"): + assert write_reading_failure(_refusal(), blocked) is None + assert write_reading_cost(blocked, complete=False) is False + monkeypatch.setitem(sys.modules, "fi.alk.harness.hosted_entrypoint", None) + assert reading_entry._read_secret_values((tmp_path / "unread.json",)) == () + + traced = [ + record.getMessage() + for record in caplog.records + if record.levelname == "WARNING" and (record.exc_info or record.exc_text) + ] + for said in ("failure file not written", "spend file not written", "secret values not read"): + assert any(said in line for line in traced), (said, traced) + + def test_cost_json_keeps_the_build_keys_and_adds_complete(tmp_path): spend.record( "read-agent", @@ -534,3 +657,26 @@ def test_an_unpriced_turn_marks_the_cost_incomplete(tmp_path): assert cost["unpriced_turns"] == 1 assert cost["total_usd"] == 0.1 assert cost["complete"] is False + + +def test_a_failed_cost_write_leaves_no_half_file(tmp_path, monkeypatch): + output = tmp_path / "reading" + output.mkdir() + + def refuses(*_args, **_kwargs): + raise OSError("no space left on the device") + + monkeypatch.setattr(os, "replace", refuses) + + assert write_reading_cost(output, complete=False) is False + assert list(output.iterdir()) == [] + + +def test_a_builds_spend_file_gains_no_complete_key(tmp_path): + assert write_reading_cost(tmp_path / "reading", complete=True) is True + build = tmp_path / "authoring" / "cost.json" + + spend.journal_to(build) + spend.record("understand-agent", 0.1, turns=1, models={"m"}) + + assert json.loads(build.read_text(encoding="utf-8")) == spend.snapshot() From bd198d59927e1a4c8d1bdb1bf941c970643f775a Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Fri, 6 Nov 2026 14:35:53 +0530 Subject: [PATCH 04/23] fix(harness): open a reading's stage as the build opens its own - One helper, understand.reading_settings, chooses the stage's turns, thinking and setup-contract gate for both the build and the reading. - A generic job's reading now refuses what the build's gate refuses. TH-8951 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_reading.py | 16 ++-- src/fi/alk/harness/cli.py | 14 +--- src/fi/alk/harness/understand.py | 17 +++++ tests/harness/test_agent_reading.py | 109 +++++++++++++++++++++++++--- 4 files changed, 121 insertions(+), 35 deletions(-) diff --git a/src/fi/alk/harness/agent_reading.py b/src/fi/alk/harness/agent_reading.py index 1b929a739..f14bda492 100644 --- a/src/fi/alk/harness/agent_reading.py +++ b/src/fi/alk/harness/agent_reading.py @@ -20,8 +20,6 @@ from . import session from .agent_type_check import EXIT_REFUSED from .backends import StageDone -from .builder import BUILDER_TURNS -from .config import chosen_model from .failure_file import FAILURE_FILE from .job import FailureDomain, HarnessFailure, HarnessJob, SourceKind from .reading_entry import ( @@ -45,7 +43,7 @@ from .session import Stage, StageIdleTimeout from .source_folder import refuse_unusable_folder from .sources import resolve as resolve_source -from .understand import SUBMIT_REMINDER, open_stage, opening +from .understand import SUBMIT_REMINDER, open_stage, opening, reading_settings logger = logging.getLogger(__name__) logger.addFilter(MaskSecrets()) @@ -230,15 +228,11 @@ async def _reading( root=source, subfolder=job.source.subfolder, ) - offered = job.metadata.get("available_evals") - opened, _ = open_stage( - agent_source, - out=output, - available_evals=offered if isinstance(offered, list) else None, - ) - # The build's settings, not the stage's own: the reading is meant to seed a build later. + # Opened as the build opens its own reading, turns, thinking and setup rule included, so a + # contract saved here is one the build's gate would have kept. + opened, _ = open_stage(agent_source, out=output, **reading_settings(job)) stage = _ReadingStage( - replace(opened.spec, max_turns=BUILDER_TURNS, model=chosen_model(), thinking=False), + opened.spec, name=opened.name, backend=backend, output=output, diff --git a/src/fi/alk/harness/cli.py b/src/fi/alk/harness/cli.py index 7bca093d5..674de7d8e 100644 --- a/src/fi/alk/harness/cli.py +++ b/src/fi/alk/harness/cli.py @@ -39,7 +39,7 @@ from .session import TEXT, Event from .sessions import Session, new_id, save as save_session from .sources import resolve, supported -from .understand import SUBMIT_REMINDER, load, open_stage, opening +from .understand import SUBMIT_REMINDER, load, open_stage, opening, reading_settings from .world.snapshot import saved as world_saved @@ -287,23 +287,13 @@ async def _understand(args: argparse.Namespace) -> int: ) else: source = resolve(args.kind, name=args.name, root=args.path, subfolder=subfolder) - offered = ( - (getattr(job, "metadata", None) or {}).get("available_evals") if job else None - ) - generic = bool( - job is not None - and isinstance(getattr(job, "metadata", None), dict) - and job.metadata.get("generic_harness_v1") is True - ) stage, destination = open_stage( source, out=Path(args.out) if args.out else None, # Unattended, there is nobody to answer, so the model records what it could not # resolve in open_questions rather than blocking on a prompt nobody will see. ask=permission_gate(_ask_operator) if args.interactive else None, - available_evals=offered if isinstance(offered, list) else None, - require_setup_contract=generic and not args.interactive, - max_turns=140 if generic and not args.interactive else 70, + **reading_settings(job, interactive=args.interactive), ) print(f"agent: {source.name} ({source.kind})") diff --git a/src/fi/alk/harness/understand.py b/src/fi/alk/harness/understand.py index 976a6c4bb..dddf12036 100644 --- a/src/fi/alk/harness/understand.py +++ b/src/fi/alk/harness/understand.py @@ -111,6 +111,23 @@ def _eval_catalogue_briefing(available_evals: list[dict[str, Any]] | None) -> st ) +def reading_settings(job: Any, *, interactive: bool = False) -> dict[str, Any]: + """The ``open_stage`` arguments a job's agent is read with, by a build or by a reading alone. + + One choice for both, so a contract a reading saves is one the build's gate would keep. A + generic job read unattended must declare its setup, which takes more turns. + """ + metadata = getattr(job, "metadata", None) if job is not None else None + metadata = metadata if isinstance(metadata, dict) else {} + offered = metadata.get("available_evals") + generic = metadata.get("generic_harness_v1") is True and not interactive + return { + "available_evals": offered if isinstance(offered, list) else None, + "require_setup_contract": generic, + "max_turns": 140 if generic else 70, + } + + def open_stage( source: AgentSource, *, diff --git a/tests/harness/test_agent_reading.py b/tests/harness/test_agent_reading.py index f68fef632..0be4bc640 100644 --- a/tests/harness/test_agent_reading.py +++ b/tests/harness/test_agent_reading.py @@ -6,6 +6,7 @@ from __future__ import annotations +import argparse import asyncio import json import logging @@ -28,14 +29,13 @@ spend, ) from fi.alk.harness.backends import StageDone -from fi.alk.harness.builder import BUILDER_TURNS from fi.alk.harness.config import chosen_model from fi.alk.harness.contract import AgentContract from fi.alk.harness.framework_detection import detect_framework from fi.alk.harness.job import HarnessJob, SourceKind from fi.alk.harness.reading_entry import READING_JOB_INVALID, run_reading_mode -from fi.alk.harness.tools import CONTRACT_SERVER -from fi.alk.harness.understand import SUBMIT_REMINDER +from fi.alk.harness.tools import CONTRACT_SERVER, _checked_contract +from fi.alk.harness.understand import SUBMIT_REMINDER, reading_settings from fi.alk.providers.agent import build_authored_agent EXAMPLES = Path(__file__).resolve().parents[2] / "examples" / "frameworks" @@ -284,6 +284,17 @@ def opened(*args, **kwargs): monkeypatch.setattr(agent_reading, "open_stage", opened) +def _few_turns(monkeypatch, turns: int) -> None: + real = agent_reading.open_stage + + def opened(*args, **kwargs): + stage, destination = real(*args, **kwargs) + stage.spec.max_turns = turns + return stage, destination + + monkeypatch.setattr(agent_reading, "open_stage", opened) + + def _short_limit(monkeypatch, soft: float = 0.3) -> None: monkeypatch.setenv(_LIMIT_ENV, str(soft)) @@ -779,7 +790,7 @@ def test_a_gemini_call_limit_is_incomplete_with_no_reminder(tmp_path, model, mon sent, calls = _real_gemini( monkeypatch, lambda who, calls: _gemini_calls("Glob", {"pattern": "*.py"}, len(calls)) ) - monkeypatch.setattr(agent_reading, "BUILDER_TURNS", 3) + _few_turns(monkeypatch, 3) _repo(tmp_path) output = tmp_path / "reading" @@ -814,7 +825,7 @@ def reply(who, calls): return _gemini_calls("Glob", {"pattern": "*.py"}, len(calls)) sent, calls = _real_gemini(monkeypatch, reply) - monkeypatch.setattr(agent_reading, "BUILDER_TURNS", 6) + _few_turns(monkeypatch, 6) _repo(tmp_path) output = tmp_path / "reading" @@ -844,7 +855,7 @@ def test_an_earlier_failed_turn_is_still_a_model_error(tmp_path, model): def test_a_cut_on_the_last_allowed_call_is_not_a_turn_limit(tmp_path, model): _repo(tmp_path) fake = model( - _ends(outcome="failed", is_error=True, turns=BUILDER_TURNS, errors=["connection reset"]), + _ends(outcome="failed", is_error=True, turns=70, errors=["connection reset"]), _ends(), name="vertex-gemini", ) @@ -1360,20 +1371,94 @@ def test_a_forbidden_model_is_not_retried(tmp_path, model): _EVALS = [{"name": "tone_check", "description": "Checks the tone.", "modality": "chat"}] -def test_the_reading_uses_the_builders_settings(tmp_path, model): +class _Opened(Exception): + pass + + +@pytest.mark.parametrize( + ("metadata", "interactive", "expected"), + [ + ( + {"generic_harness_v1": True, "available_evals": _EVALS}, + False, + {"available_evals": _EVALS, "require_setup_contract": True, "max_turns": 140}, + ), + ( + {"available_evals": _EVALS}, + False, + {"available_evals": _EVALS, "require_setup_contract": False, "max_turns": 70}, + ), + ( + {"generic_harness_v1": True}, + True, + {"available_evals": None, "require_setup_contract": False, "max_turns": 70}, + ), + ], + ids=["generic", "not_generic", "generic_with_an_operator"], +) +def test_the_build_opens_its_reading_as_it_always_has( + tmp_path, monkeypatch, metadata, interactive, expected +): + from fi.alk.harness import cli + + job = HarnessJob.model_validate(_job(metadata=metadata)) + assert reading_settings(job, interactive=interactive) == expected + seen: dict = {} + + def opened(_source, **kwargs): + seen.update(kwargs) + raise _Opened + + monkeypatch.setattr(cli, "open_stage", opened) + args = argparse.Namespace( + job=job, + kind="repo", + name="agent", + path=str(_repo(tmp_path)), + out=str(tmp_path / "out"), + interactive=interactive, + ) + + with pytest.raises(_Opened): + asyncio.run(cli._understand(args)) + + assert {key: seen[key] for key in expected} == expected + + +@pytest.mark.parametrize(("generic", "turns"), [(False, 70), (True, 140)]) +def test_the_reading_opens_as_the_build_does(tmp_path, model, generic, turns): _repo(tmp_path) fake = model() + metadata = {"available_evals": _EVALS, **({"generic_harness_v1": True} if generic else {})} - _run(tmp_path, _job(metadata={"available_evals": _EVALS})) + _run(tmp_path, _job(metadata=metadata)) [spec] = fake.created - assert spec.max_turns == BUILDER_TURNS == 200 + assert spec.max_turns == turns assert spec.model == chosen_model() - assert spec.thinking is False + assert spec.thinking is True assert "tone_check" in spec.system_prompt assert CONTRACT_SERVER in spec.servers +def test_a_generic_jobs_reading_refuses_what_the_builds_gate_refuses(tmp_path, model): + # The build's gate for a generic job: a chat agent must declare where it keeps its state. + refused = _checked_contract( + dict(CHAT), source_root=None, require_setup_contract=True, subfolder=None + ) + assert isinstance(refused, dict) + assert "data_store" in refused["content"][0]["text"] + _repo(tmp_path) + fake = model(_submits(CHAT)) + output = tmp_path / "reading" + + assert _run(tmp_path, _job(metadata={"generic_harness_v1": True})) == 78 + + assert fake.submissions == 3 + assert not (output / "contract.json").exists() + assert _failure_in(output)["code"] == "agent_reading_incomplete" + + def test_the_claude_backend_accepts_the_reading_spec(tmp_path, model, monkeypatch): from fi.alk.harness.backends.claude import ClaudeBackend @@ -1386,8 +1471,8 @@ def test_the_claude_backend_accepts_the_reading_spec(tmp_path, model, monkeypatc options = ClaudeBackend().create(spec)._options try: - assert options.max_turns == 200 - assert getattr(options, "thinking", None) is None + assert options.max_turns == 70 + assert getattr(options, "thinking", None) is not None assert any(name.endswith("submit_contract") for name in options.allowed_tools) finally: prompt = options.system_prompt From ee163b04b4038c1290a8e700c4d6afe65e9971a5 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Fri, 6 Nov 2026 15:33:51 +0530 Subject: [PATCH 05/23] fix(harness): pin a reading's failure paths, and mask what reaches its logs - Tests pin the time-limit and crash codes, and that cost.json ends incomplete after a timeout or a crash the work outlived. - After the work returns, the limit still ends a process a leftover thread keeps alive, with the work's own code and no new writes. - A job error's location drops mapping keys; the mask also covers a secret's lines, repr and escaped forms; secrets are read per reading. - The reading's opening shares the build's guidance; names used across modules are public; a dead attribute is gone. TH-8951 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_reading.py | 49 ++++++----- src/fi/alk/harness/cli.py | 12 +-- src/fi/alk/harness/reading_entry.py | 127 +++++++++++++++++++++------ src/fi/alk/harness/reading_result.py | 22 ++--- tests/harness/test_agent_reading.py | 71 +++++++++++++-- tests/harness/test_reading_entry.py | 122 ++++++++++++++++++++++--- 6 files changed, 319 insertions(+), 84 deletions(-) diff --git a/src/fi/alk/harness/agent_reading.py b/src/fi/alk/harness/agent_reading.py index f14bda492..ee017adf6 100644 --- a/src/fi/alk/harness/agent_reading.py +++ b/src/fi/alk/harness/agent_reading.py @@ -20,6 +20,7 @@ from . import session from .agent_type_check import EXIT_REFUSED from .backends import StageDone +from .cli import guidance_text from .failure_file import FAILURE_FILE from .job import FailureDomain, HarnessFailure, HarnessJob, SourceKind from .reading_entry import ( @@ -34,11 +35,11 @@ ) from .reading_result import ( CONTRACT_FILE, - _classify, - _document, - _model_error, - _out_of_turns, - _saved, + classify, + document, + model_error, + out_of_turns, + saved_file, ) from .session import Stage, StageIdleTimeout from .source_folder import refuse_unusable_folder @@ -89,14 +90,12 @@ def __init__( *, name: str, backend: Any, - output: Path, running_totals: bool, ) -> None: # Never replayed after a quiet stream: a replayed turn is paid for again, and a new # session would restart Claude's running totals under the baseline the difference is # taken from. super().__init__(spec, name=name, backend=backend, idle_retries=0) - self.output = output self.running_totals = running_totals self.done: list[StageDone] = [] self.in_flight = False @@ -147,21 +146,21 @@ def _source_has_files(source: Path) -> bool: def _reminder_due(stage: _ReadingStage, output: Path, stopping: asyncio.Event) -> bool: - if _saved(output) or stopping.is_set(): + if saved_file(output) or stopping.is_set(): return False opened = stage.done[-1] if stage.done else None # An error flag with no status may have passed; one nudge costs less than a whole retry. return opened is None or ( - opened.api_error_status is None and not _out_of_turns(opened) + opened.api_error_status is None and not out_of_turns(opened) ) async def _converse( - stage: _ReadingStage, agent_source: Any, output: Path, stopping: asyncio.Event + stage: _ReadingStage, message: str, output: Path, stopping: asyncio.Event ) -> None: # The session lives inside this task, so cancelling the task closes it. async with stage: - await stage.say(opening(agent_source)) + await stage.say(message) if _reminder_due(stage, output, stopping): await stage.say(SUBMIT_REMINDER) @@ -235,11 +234,13 @@ async def _reading( opened.spec, name=opened.name, backend=backend, - output=output, running_totals=getattr(backend, "name", "") == "claude", ) stopping = asyncio.Event() - task = asyncio.create_task(_converse(stage, agent_source, output, stopping)) + # The build's opening, the job's guidance included: a generic job's runtime advice changes + # what the contract records. + message = opening(agent_source) + guidance_text(job.authoring_guidance()) + task = asyncio.create_task(_converse(stage, message, output, stopping)) finished, _ = await asyncio.wait( {task}, timeout=max(0.0, started + soft_limit - time.monotonic()) ) @@ -247,11 +248,11 @@ async def _reading( if not finished: await _stop(stage, task, stopping, wait) # A request cut off by the stop may be billed without being in the last total. - saved = _saved(output) + saved = saved_file(output) if saved == FAILURE_FILE: return EXIT_REFUSED, False if saved == CONTRACT_FILE: - return _document(job, source, output), False + return document(job, source, output), False return _timed_out(output, soft_limit), False error = task.exception() @@ -260,13 +261,13 @@ async def _reading( raise error if isinstance(error, StageIdleTimeout): # The quiet turn's spend never arrived; what was saved before it still stands. - saved = _saved(output) + saved = saved_file(output) if saved == FAILURE_FILE: return EXIT_REFUSED, False if saved == CONTRACT_FILE: - return _document(job, source, output), False + return document(job, source, output), False return ( - _model_error( + model_error( output, "The model stopped responding before the agent's description was saved " f"({error}).", @@ -279,11 +280,11 @@ async def _reading( error is None and not stage.in_flight and not any( - done.is_error and done.api_error_status is None and not _out_of_turns(done) + done.is_error and done.api_error_status is None and not out_of_turns(done) for done in stage.done ) ) - return _classify(job, source, output, stage, error), complete + return classify(job, source, output, stage, error), complete async def _read(job: HarnessJob, source: Path, output: Path, *, interrupt_wait: float) -> int: @@ -298,10 +299,12 @@ async def _read(job: HarnessJob, source: Path, output: Path, *, interrupt_wait: ) except Exception as exc: # A contract the gate saved is paid for; a crash would have the platform pay again. - if _saved(output) != CONTRACT_FILE: + if saved_file(output) != CONTRACT_FILE: raise - logger.warning("reading fault after a saved contract: class=%s", type(exc).__name__) - code, complete = _document(job, Path(source), output), False + logger.warning( + "reading fault after a saved contract: class=%s", type(exc).__name__, exc_info=True + ) + code, complete = document(job, Path(source), output), False finally: # The work owns the final cost file, on every way out. write_reading_cost(output, complete=complete) diff --git a/src/fi/alk/harness/cli.py b/src/fi/alk/harness/cli.py index 674de7d8e..31c36a086 100644 --- a/src/fi/alk/harness/cli.py +++ b/src/fi/alk/harness/cli.py @@ -15,6 +15,7 @@ import sys import time import uuid +from collections.abc import Iterable from pathlib import Path from typing import Any @@ -168,11 +169,12 @@ async def _ask_operator(_tool_name: str, payload: dict[str, Any], _context: Any) def _guidance(args: argparse.Namespace) -> str: - instructions = [ - str(item).strip() - for item in (getattr(args, "guidance", None) or []) - if str(item).strip() - ] + return guidance_text(getattr(args, "guidance", None) or []) + + +def guidance_text(guidance: Iterable[Any]) -> str: + """A job's guidance as every stage that reads it is told it; empty when there is none.""" + instructions = [str(item).strip() for item in guidance if str(item).strip()] if not instructions: return "" return ( diff --git a/src/fi/alk/harness/reading_entry.py b/src/fi/alk/harness/reading_entry.py index 40812677d..400f1578b 100644 --- a/src/fi/alk/harness/reading_entry.py +++ b/src/fi/alk/harness/reading_entry.py @@ -16,13 +16,14 @@ import math import os import threading -from collections.abc import Awaitable, Callable +import typing +from collections.abc import Awaitable, Callable, Iterator from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Any -from pydantic import ValidationError +from pydantic import BaseModel, ValidationError from . import spend from .agent_type_check import EXIT_REFUSED @@ -92,10 +93,34 @@ def _read_secret_values(paths: tuple[Path, ...]) -> tuple[str, ...]: try: from .hosted_entrypoint import peek_secret_values - return tuple(value for path in paths for value in peek_secret_values(path)) + values = {form for path in paths for value in peek_secret_values(path) for form in _forms(value)} except Exception: _secrets_logger.warning("secret values not read; only URL userinfo is masked", exc_info=True) return () + # Longest first, so a whole value is masked before any of its parts. + return tuple(sorted(values, key=len, reverse=True)) + + +def _forms(value: str) -> Iterator[str]: + """The value, and the shapes a trace can show it in: each line, each string inside it when it + is JSON (a service account's key), and its escaped text as ``repr`` and JSON print it.""" + yield value + yield from value.splitlines() + yield repr(value)[1:-1] + yield json.dumps(value)[1:-1] + try: + parsed = json.loads(value) + except ValueError: + return + pending = [parsed] + while pending: + item = pending.pop() + if isinstance(item, str): + yield from _forms(item) + elif isinstance(item, dict): + pending.extend(item.values()) + elif isinstance(item, list): + pending.extend(item) def _masked(text: str) -> str: @@ -162,12 +187,38 @@ def reading_failure( ) +def _models_in(annotation: Any) -> Iterator[type[BaseModel]]: + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + yield annotation + for inner in typing.get_args(annotation): + yield from _models_in(inner) + + +@functools.cache +def _job_field_names() -> frozenset[str]: + names: set[str] = set() + seen: set[type[BaseModel]] = set() + pending: list[type[BaseModel]] = [HarnessJob] + while pending: + model = pending.pop() + if model in seen: + continue + seen.add(model) + for name, field in model.model_fields.items(): + names.update(filter(None, (name, field.alias))) + pending.extend(_models_in(field.annotation)) + return frozenset(names) + + def _unreadable_job(exc: Exception) -> str: # Only where and how it failed: pydantic's own text repeats the job, and a job may hold a token. if not isinstance(exc, ValidationError): return type(exc).__name__ + fields = _job_field_names() found = "; ".join( - f"{'.'.join(str(part) for part in error['loc']) or 'job'}: {error['type']}" + # A dict's keys are the job's own data, and may be a token, so only field names and + # indexes are shown. + f"{'.'.join(str(part) if isinstance(part, int) or part in fields else '*' for part in error['loc']) or 'job'}: {error['type']}" for error in exc.errors(include_input=False) ) return f"ValidationError: {found}" @@ -183,14 +234,35 @@ def _output_in_use(output: Path) -> bool: return True -def _start_hard_limit( - mode: ReadingMode, output: Path, limit: float | None -) -> threading.Timer | None: - if limit is None: - return None - exit_code = EXIT_RETRYABLE if mode.timed_out_retryable else EXIT_REFUSED - - def end_the_process() -> None: +class _HardLimit: + """Ends the process at the limit. Before the work returns, a timeout is written first; after + it, the work's files stand and the process ends with the work's own code.""" + + def __init__(self, mode: ReadingMode, output: Path, limit: float) -> None: + self._mode = mode + self._output = output + self._limit = limit + self._returned: int | None = None + self._timer = threading.Timer(limit, self._end_the_process) + self._timer.daemon = True + self._timer.start() + + def returned(self, code: int) -> None: + self._returned = code + + def cancel(self) -> None: + self._timer.cancel() + + def _end_the_process(self) -> None: + returned = self._returned + if returned is not None: + # Only a thread the work left behind keeps the process alive now. Its files are on + # disk, so nothing is written over them. + os._exit(returned) + mode, output, limit = self._mode, self._output, self._limit + exit_code = EXIT_RETRYABLE if mode.timed_out_retryable else EXIT_REFUSED + # Nothing logs on the way out: a blocked stderr must not hold off the exit. + logging.disable(logging.CRITICAL) try: write_reading_failure( reading_failure( @@ -207,26 +279,22 @@ def end_the_process() -> None: # Work running in a thread can't be cancelled, and a normal exit would wait for it. os._exit(exit_code) - timer = threading.Timer(limit, end_the_process) - timer.daemon = True - timer.start() - return timer - async def _work_then_stop_the_clock( - mode: ReadingMode, job: HarnessJob, source: Path, output: Path, timer: threading.Timer | None + mode: ReadingMode, job: HarnessJob, source: Path, output: Path, limit: _HardLimit | None ) -> int: - try: - return await mode.work(job, source, output) - finally: - # Inside the loop: asyncio.run waits for leftover worker threads before it returns, and - # the limit must not fire during that wait over a reading that has already ended. - if timer is not None: - timer.cancel() + code = await mode.work(job, source, output) + # Told inside the loop: asyncio.run waits for leftover worker threads before it returns, and + # the limit must not write a timeout over a reading that has already ended. + if limit is not None: + limit.returned(code) + return code def run_reading_mode(mode: ReadingMode, *, job_path: Path, source: Path, output: Path) -> int: """Refuse what can't be read, start the cost at zero, then run the mode's work.""" + # Read afresh for each reading, so one job's secrets never stand in for another's. + _read_secret_values.cache_clear() _read_secret_values(_SECRET_FILES) output = Path(output) @@ -291,18 +359,19 @@ def run_reading_mode(mode: ReadingMode, *, job_path: Path, source: Path, output: return EXIT_RETRYABLE # Read now, not when the mode was made, so it is the same limit the work reads as it starts. - timer = _start_hard_limit(mode, output, _hard_limit(mode)) + seconds = _hard_limit(mode) + limit = None if seconds is None else _HardLimit(mode, output, seconds) crash: Exception | None = None try: logger.info("reading started at %s", datetime.now(timezone.utc).isoformat()) - code = asyncio.run(_work_then_stop_the_clock(mode, job, source, output, timer)) + code = asyncio.run(_work_then_stop_the_clock(mode, job, source, output, limit)) except Exception as exc: logger.error("the %s reading crashed", mode.name, exc_info=True) crash = exc finally: # Cancelled first, so the limit can't fire during a crash's writes or the caller's cleanup. - if timer is not None: - timer.cancel() + if limit is not None: + limit.cancel() if crash is not None: write_reading_failure( diff --git a/src/fi/alk/harness/reading_result.py b/src/fi/alk/harness/reading_result.py index a4940d058..1b5dab92e 100644 --- a/src/fi/alk/harness/reading_result.py +++ b/src/fi/alk/harness/reading_result.py @@ -49,7 +49,7 @@ _PASSING_STATUSES = frozenset({408, 409, 429}) -def _saved(output: Path) -> str: +def saved_file(output: Path) -> str: if (output / FAILURE_FILE).is_file(): return FAILURE_FILE if (output / CONTRACT_FILE).is_file(): @@ -57,7 +57,7 @@ def _saved(output: Path) -> str: return "" -def _out_of_turns(done: StageDone) -> bool: +def out_of_turns(done: StageDone) -> bool: # ADK enforces its call budget by raising, which the Gemini backend reports as a failure with # no status, so ADK's own words are the signal. Matched whole, since the phrase inside other # text may be the model's own. The count is no signal: it catches no real limit, and it would @@ -78,7 +78,7 @@ def _refusing(status: Any) -> bool: ) -def _model_error(output: Path, message: str, status: Any = None, *, retryable: bool) -> int: +def model_error(output: Path, message: str, status: Any = None, *, retryable: bool) -> int: write_reading_failure( reading_failure( FailureDomain.INFRASTRUCTURE, @@ -93,23 +93,23 @@ def _model_error(output: Path, message: str, status: Any = None, *, retryable: b return EXIT_RETRYABLE if retryable else EXIT_REFUSED -def _classify( +def classify( job: HarnessJob, source: Path, output: Path, stage: _ReadingStage, error: BaseException | None, ) -> int: - saved = _saved(output) + saved = saved_file(output) if saved == FAILURE_FILE: return EXIT_REFUSED # Before any error: the gate accepted the reading, and refusing it now would pay for it twice. if saved == CONTRACT_FILE: - return _document(job, source, output) + return document(job, source, output) forbidden = stage.forbidden_models() if forbidden: # A retry would bill the same model again. - return _model_error( + return model_error( output, "The provider billed a model this harness may not spend on (" + ", ".join(sorted(forbidden)) @@ -118,12 +118,12 @@ def _classify( ) # A last turn out of turns spent a whole budget, so a retry would likely spend another and # stop the same way, whatever failed before it. - ran_out = bool(stage.done) and _out_of_turns(stage.done[-1]) + ran_out = bool(stage.done) and out_of_turns(stage.done[-1]) failed = [ done for done in stage.done if done.api_error_status is not None - or (done.is_error and not _out_of_turns(done)) + or (done.is_error and not out_of_turns(done)) ] if not ran_out and (failed or error is not None): status = next( @@ -134,7 +134,7 @@ def _classify( if error is not None: said.append(f"{type(error).__name__}: {error}") shown = f" (status {status})" if isinstance(status, int) else "" - return _model_error( + return model_error( output, f"The model provider failed while the agent was being read{shown}: " + ("; ".join(said) or "no detail was given."), @@ -182,7 +182,7 @@ def _write_whole(path: Path, body: bytes) -> None: raise -def _document(job: HarnessJob, source: Path, output: Path) -> int: +def document(job: HarnessJob, source: Path, output: Path) -> int: """The agent's description from the saved contract, as a build makes it.""" # One catch over every step: a saved contract is paid for, so nothing after it is retryable. try: diff --git a/tests/harness/test_agent_reading.py b/tests/harness/test_agent_reading.py index 0be4bc640..192eaf997 100644 --- a/tests/harness/test_agent_reading.py +++ b/tests/harness/test_agent_reading.py @@ -103,7 +103,7 @@ async def send(self, message: str) -> None: async def interrupt(self) -> bool: self.backend.interrupts.append(time.monotonic()) if self.backend.interrupt_raises: - raise RuntimeError("the interrupt control request timed out") + raise RuntimeError(self.backend.interrupt_says) self.interrupted.set() return True @@ -125,6 +125,7 @@ def __init__(self, name: str, output: Path, turns: list) -> None: self.stopped: list[dict] = [] self.seen: list = [] self.interrupt_raises = False + self.interrupt_says = "the interrupt control request timed out" self.start_raises = False self.saves_on_start: dict | None = None self.submissions = 0 @@ -1033,6 +1034,29 @@ def test_an_interrupt_that_raises_is_still_a_timeout(tmp_path, model, monkeypatc assert "RuntimeError: the interrupt control request timed out" in logged.exc_text +def test_the_readings_own_log_is_masked(tmp_path, model, monkeypatch, caplog): + secret = "sk-provider-0123456789" + secrets = tmp_path / "secrets.json" + secrets.write_text(json.dumps({"TARGET_KEY": secret}), encoding="utf-8") + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (secrets,)) + _repo(tmp_path) + _short_limit(monkeypatch) + fake = model(_quiet) + fake.interrupt_raises = True + fake.interrupt_says = f"the provider refused key {secret}" + + with caplog.at_level("WARNING", logger=agent_reading.__name__): + assert _run(tmp_path, wait=0.3) == 78 + + [logged] = [ + record + for record in caplog.records + if record.name == agent_reading.__name__ and "not delivered" in record.getMessage() + ] + assert "RuntimeError: the provider refused key ***" in logged.exc_text + assert secret not in caplog.text + + def test_a_session_that_breaks_after_the_stop_is_logged_and_still_a_timeout( tmp_path, model, monkeypatch, caplog ): @@ -1393,8 +1417,14 @@ class _Opened(Exception): True, {"available_evals": None, "require_setup_contract": False, "max_turns": 70}, ), + ( + # Only the flag itself, not a value that merely reads as true. + {"generic_harness_v1": "true"}, + False, + {"available_evals": None, "require_setup_contract": False, "max_turns": 70}, + ), ], - ids=["generic", "not_generic", "generic_with_an_operator"], + ids=["generic", "not_generic", "generic_with_an_operator", "flag_as_text"], ) def test_the_build_opens_its_reading_as_it_always_has( tmp_path, monkeypatch, metadata, interactive, expected @@ -1441,6 +1471,25 @@ def test_the_reading_opens_as_the_build_does(tmp_path, model, generic, turns): assert CONTRACT_SERVER in spec.servers +def test_the_reading_opens_with_the_builds_words_and_the_jobs_guidance(tmp_path, model): + from fi.alk.harness import cli + from fi.alk.harness.sources import resolve + from fi.alk.harness.understand import opening + + source = _repo(tmp_path) + fake = model() + raw = _job(metadata={"generic_harness_v1": True}) + + _run(tmp_path, raw) + + job = HarnessJob.model_validate(raw) + built = opening(resolve("repo", name=source.name, root=source)) + cli._guidance( + argparse.Namespace(guidance=job.authoring_guidance()) + ) + assert fake.sent[0] == built + assert "standalone Dockerfile cannot be built" in fake.sent[0] + + def test_a_generic_jobs_reading_refuses_what_the_builds_gate_refuses(tmp_path, model): # The build's gate for a generic job: a chat agent must declare where it keeps its state. refused = _checked_contract( @@ -1607,6 +1656,11 @@ def test_a_good_time_limit_is_read(monkeypatch): # The helper's limit is the reading's own plus its margin, and a timeout repeats if retried. assert reading_entry._hard_limit(mode) == 120.0 + 180.0 assert mode.timed_out_retryable is False + # The contract's codes, as the platform reads them. + assert (mode.timed_out_code, mode.crashed_code) == ( + "agent_reading_timed_out", + "agent_reading_crashed", + ) def test_a_fault_after_a_saved_contract_still_leaves_the_document(tmp_path, model, caplog): @@ -1623,10 +1677,15 @@ def test_a_fault_after_a_saved_contract_still_leaves_the_document(tmp_path, mode assert _names(output) == {"contract.json", "agent.json", "cost.json"} assert fake.sent == [] _assert_cost(output, 0.0, False) - # The fault is named by its class only: its text may carry what the customer's code said. - logged = [record.getMessage() for record in caplog.records if record.name == agent_reading.__name__] - assert any("FileNotFoundError" in line for line in logged) - assert not any("was not found" in line for line in logged) + # The line names the class; the trace, masked like every line here, keeps the rest. + [logged] = [ + record + for record in caplog.records + if record.name == agent_reading.__name__ and "saved contract" in record.getMessage() + ] + assert "FileNotFoundError" in logged.getMessage() + assert "was not found" not in logged.getMessage() + assert "FileNotFoundError: the Claude CLI was not found" in logged.exc_text def test_a_session_that_cannot_open_is_a_crash(tmp_path, model): diff --git a/tests/harness/test_reading_entry.py b/tests/harness/test_reading_entry.py index f88965fd1..19bd049ef 100644 --- a/tests/harness/test_reading_entry.py +++ b/tests/harness/test_reading_entry.py @@ -132,7 +132,7 @@ def _refusal(code: str = "scan_job_invalid") -> HarnessFailure: # The helper ends the process with os._exit at its limit, so those tests run it in a child. _CHILD = r""" -import asyncio, json, logging, sys, time +import asyncio, json, logging, shutil, sys, time config = json.loads(sys.argv[1]) sys.path[:0] = config["sys_path"] logging.basicConfig(level=logging.INFO) @@ -153,12 +153,22 @@ async def work(job, source, output): return 0 if config["work"] == "leaves_a_thread": (output / "agent.json").write_text('{"agent": "kept"}') + reading_entry.write_reading_cost(output, complete=True) try: # A scan given up on by its caller: the thread itself runs on past the limit. - await asyncio.wait_for(asyncio.to_thread(time.sleep, 3.0), 0.1) + await asyncio.wait_for(asyncio.to_thread(time.sleep, 30.0), 0.1) except TimeoutError: pass return 0 + if config["work"] in ("completes_then_blocks", "completes_then_raises"): + spend.record("read-agent", 0.3, turns=1, models={"m"}) + reading_entry.write_reading_cost(output, complete=True) + if config["work"] == "completes_then_raises": + raise RuntimeError("the work broke after its last write") + if config["work"] == "spoils_the_output_then_blocks": + # Every write at the limit now fails, and each failure would log. + shutil.rmtree(output) + output.write_text("a file, not a folder") if config["work"] == "records_then_blocks": spend.record("read-agent", 0.21, turns=1, models={"m"}) if config["work"] == "breaks_the_writers_then_blocks": @@ -242,10 +252,26 @@ def test_an_unreadable_job_is_refused_before_any_work(tmp_path, values, case): assert work.calls == [] -def test_a_refused_job_leaves_none_of_its_values_anywhere(tmp_path, values, capfd, caplog): - token = "ghp_REVIEWSECRET1234567890abcdef" +_TOKEN = "ghp_REVIEWSECRET1234567890abcdef" + + +@pytest.mark.parametrize( + "refused", + [ + {**_job(), "metadata": {"token": _TOKEN}}, + # pydantic's own message for this one repeats the value whole. + {**_job(), "schema_version": _TOKEN}, + # A dict's key is in the error's location. + {**_job(), "agent": {"connector": "auto", "secret_refs": {_TOKEN: "not-an-object"}}}, + ], + ids=["secret_field", "schema_version", "secret_refs_key"], +) +def test_a_refused_job_leaves_none_of_its_values_anywhere( + tmp_path, values, capfd, caplog, refused +): + token = _TOKEN job_path = tmp_path / "job.json" - job_path.write_text(json.dumps({**_job(), "metadata": {"token": token}}), encoding="utf-8") + job_path.write_text(json.dumps(refused), encoding="utf-8") with caplog.at_level("DEBUG"): assert _run(_mode(values, _Work()), tmp_path, job_path) == 78 @@ -443,16 +469,56 @@ def test_a_limit_named_by_the_environment_is_read_when_the_reading_starts( assert "past its 1 s limit" in failure["message"] -def test_a_thread_left_after_the_work_does_not_time_the_reading_out(tmp_path): +def test_a_thread_left_after_the_work_ends_the_process_quietly_at_the_limit(tmp_path): mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} finished, elapsed = _run_in_a_child(tmp_path, mode_values, "leaves_a_thread") + output = tmp_path / "reading" + # The work's own code, at the limit, not after the 30 s thread; and nothing written over the + # files the work left. assert finished.returncode == 0, finished.stderr - # The process did wait for the thread, well past the limit, and the limit stayed quiet. - assert elapsed >= 3.0 - assert not (tmp_path / "reading" / "failure.json").exists() - assert (tmp_path / "reading" / "agent.json").read_text() == '{"agent": "kept"}' + assert 1.0 <= elapsed <= 1.0 + 5 + assert not (output / "failure.json").exists() + assert (output / "agent.json").read_text() == '{"agent": "kept"}' + assert _cost_in(output) == {**_empty_spend(), "complete": True} + + +def _empty_spend() -> dict: + return {"schema": "futureagi.harness-spend.v1", "total_usd": 0.0, "unpriced_turns": 0, "stages": []} + + +def test_a_timeout_after_a_complete_cost_marks_it_incomplete(tmp_path): + mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} + + finished, _ = _run_in_a_child(tmp_path, mode_values, "completes_then_blocks") + + assert finished.returncode == 78, finished.stderr + cost = _cost_in(tmp_path / "reading") + assert (cost["total_usd"], cost["complete"]) == (0.3, False) + assert _failure_in(tmp_path / "reading")["code"] == "agent_reading_timed_out" + + +def test_a_crash_after_a_complete_cost_marks_it_incomplete(tmp_path): + finished, _ = _run_in_a_child(tmp_path, READ_AGENT_VALUES, "completes_then_raises") + + assert finished.returncode == 1, finished.stderr + cost = _cost_in(tmp_path / "reading") + assert (cost["total_usd"], cost["complete"]) == (0.3, False) + assert _failure_in(tmp_path / "reading")["code"] == "agent_reading_crashed" + + +def test_the_way_out_at_the_limit_logs_nothing(tmp_path): + mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} + + finished, elapsed = _run_in_a_child(tmp_path, mode_values, "spoils_the_output_then_blocks") + + assert finished.returncode == 78, finished.stderr + assert elapsed <= 1.0 + 5 + # The writes did fail, and a write that fails logs everywhere else. + assert (tmp_path / "reading").is_file() + assert "reading started at " in finished.stderr + assert "not written" not in finished.stderr def test_a_crash_writes_its_code_with_no_secret_and_a_capped_message(tmp_path, values): @@ -482,6 +548,42 @@ def crash(*_args): assert write_reading_failure(_refusal(), blocked) is None +def test_a_secret_is_masked_in_the_shapes_a_trace_shows_it(tmp_path, monkeypatch, caplog): + pem = "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEF\n-----END PRIVATE KEY-----\n" + credential = json.dumps({"type": "service_account", "private_key": pem}) + secrets = tmp_path / "secrets.json" + secrets.write_text(json.dumps({"GOOGLE_CREDENTIALS_JSON": credential}), encoding="utf-8") + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (secrets,)) + + def crash(*_args): + raise RuntimeError(f"bad key {pem!r}; line MIIEvQIBADANBgkqhkiG9w0BAQEF") + + with caplog.at_level("INFO", logger=reading_entry.__name__): + assert _run(_mode(READ_AGENT_VALUES, _Work(action=crash)), tmp_path) == 1 + + kept = _failure_in(tmp_path / "reading")["message"] + caplog.text + assert "MIIEvQIBADANBgkqhkiG9w0BAQEF" not in kept + assert "BEGIN PRIVATE KEY" not in kept + # The whole value is still masked first; a value under 8 characters, or in neither file, is + # one the mask can't know. + assert reading_entry._read_secret_values((secrets,))[0] in (credential, json.dumps(credential)[1:-1]) + + +def test_each_reading_reads_the_secrets_again(tmp_path, monkeypatch): + secrets = tmp_path / "secrets.json" + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (secrets,)) + for run, value in enumerate(("first-secret-value", "second-secret-value")): + secrets.write_text(json.dumps({"KEY": value}), encoding="utf-8") + root = tmp_path / f"run{run}" + root.mkdir() + + def crash(*_args, said=value): + raise RuntimeError(f"refused {said}") + + assert _run(_mode(READ_AGENT_VALUES, _Work(action=crash)), root) == 1 + assert _failure_in(root / "reading")["message"] == "RuntimeError: refused ***" + + def test_a_value_from_the_secrets_file_is_masked(tmp_path, monkeypatch, values): target = "sk-0123456789abcdefg" simulator = "sim-0123456789abcdef" From ecda73f55bd8c6824af13523769ff50e91e66f5a Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Fri, 6 Nov 2026 15:40:30 +0530 Subject: [PATCH 06/23] test(harness): stop resetting a cache the reading helper no longer has #245 replaced reading_entry._secret_values with a functools.cache reader keyed by the secrets files. The scan tests already point those files at a fresh temporary path, so each test gets its own cache entry and there is nothing left to reset. TH-8950 Signed-off-by: khushalsonawat --- tests/harness/test_agent_scan.py | 1 - tests/harness/test_authoring_entrypoint_find_agents.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/harness/test_agent_scan.py b/tests/harness/test_agent_scan.py index cd4bfaa7d..b1a29f605 100644 --- a/tests/harness/test_agent_scan.py +++ b/tests/harness/test_agent_scan.py @@ -101,7 +101,6 @@ def _fresh_state(monkeypatch, tmp_path): monkeypatch.setattr(spend, "_path", None) # The machine's own secrets files must not decide what a test sees masked. monkeypatch.setattr(reading_entry, "_SECRET_FILES", (tmp_path / "no-secrets.json",)) - monkeypatch.setattr(reading_entry, "_secret_values", None) monkeypatch.setattr(agent_scan, "CLOCK", 1200.0) monkeypatch.setattr(agent_scan, "AI_SECONDS", 600.0) spend._stages.clear() diff --git a/tests/harness/test_authoring_entrypoint_find_agents.py b/tests/harness/test_authoring_entrypoint_find_agents.py index 709544dc8..91236af53 100644 --- a/tests/harness/test_authoring_entrypoint_find_agents.py +++ b/tests/harness/test_authoring_entrypoint_find_agents.py @@ -37,7 +37,6 @@ def _fresh_state(monkeypatch, tmp_path): monkeypatch.setattr(spend, "_path", None) # The machine's own secrets files must not decide what a test sees masked. monkeypatch.setattr(reading_entry, "_SECRET_FILES", (tmp_path / "no-secrets.json",)) - monkeypatch.setattr(reading_entry, "_secret_values", None) spend._stages.clear() yield spend._stages.clear() From ab5fc18d0e19dc8bc3349a5933fe3b0ba361ac94 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 12:55:30 +0530 Subject: [PATCH 07/23] fix(harness): keep the free agents on a model error, and let kit faults crash - Any model error now keeps the walk's rows with truncated set and exits 0; only a scan with no rows writes scan_model_error, carrying the model's own retryable flag. A retry would pay for the model again. - A fault outside any one file in the walk (run or result) now reaches the scan's crash path (scan_crashed, retryable) instead of being reported as scan_limit_reached. - Catches are narrowed to what each step expects. A file's reader faults still leave that file unread. A folder check that fails in the child sends its trace back, and the row is marked unfinished. Faults that are still absorbed (after the paid step, closing the session, tool calls) log their trace through the reading helper's MaskSecrets filter instead of the class name only. TH-8950 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_scan.py | 32 ++++----- src/fi/alk/harness/scan_ai.py | 17 ++++- src/fi/alk/harness/scan_folders.py | 37 ++++++---- src/fi/alk/harness/scan_walk.py | 104 +++++++++++++++++------------ tests/harness/test_agent_scan.py | 45 +++++++++---- tests/harness/test_scan_folders.py | 15 +++++ 6 files changed, 160 insertions(+), 90 deletions(-) diff --git a/src/fi/alk/harness/agent_scan.py b/src/fi/alk/harness/agent_scan.py index 5e6a0153b..0a5ada053 100644 --- a/src/fi/alk/harness/agent_scan.py +++ b/src/fi/alk/harness/agent_scan.py @@ -22,13 +22,14 @@ from pathlib import Path from .job import FailureDomain, HarnessFailure, HarnessJob, HarnessStage, SourceKind -from .reading_entry import ReadingMode, write_reading_cost, write_reading_failure +from .reading_entry import MaskSecrets, ReadingMode, write_reading_cost, write_reading_failure from .scan_ai import AiOutcome, run_ai_step from .scan_document import Row, ScanResultInvalid, finish, root_name, write_agents_json from .scan_folders import ClockRanOut, FolderChecker, FolderCheckerBroken from .scan_walk import _WEAKEST, Candidate, Hazard, WalkResult, describe_entry, walk_repo logger = logging.getLogger(__name__) +logger.addFilter(MaskSecrets()) # Failures, written to failure.json. SCAN_JOB_INVALID = "scan_job_invalid" @@ -292,24 +293,19 @@ def _result( if outcome is not None: if outcome.kind == "model_error": logger.warning("agent scan: model step failed, %s", outcome.message) - if outcome.retryable: - return _fail( - output, - FailureDomain.INFRASTRUCTURE, - SCAN_MODEL_ERROR, - "The model failed while looking for agents; a retry may get through.", - retryable=True, - ) if not rows: return _fail( output, FailureDomain.INFRASTRUCTURE, SCAN_MODEL_ERROR, - "The model refused the request to look for agents; a retry would be " + "The model failed while looking for agents; a retry may get through." + if outcome.retryable + else "The model refused the request to look for agents; a retry would be " "refused the same way.", - retryable=False, + retryable=outcome.retryable, ) - # The walk's rows are already checked, and a retry would be refused again. + # The walk's rows are already checked and cost nothing. A retry would pay for the + # model again to add at most what it missed, so the rows are kept, marked partial. truncated = True elif outcome.kind == "forbidden": return _fail( @@ -403,8 +399,8 @@ def _keep_what_was_paid_for( job: HarnessJob, output: Path, scan: _Scan, crash: Exception ) -> int: """After the model was paid for, a crash still ends in a file, never in a paid retry.""" - # The class only: a message could carry a path or a value from the repository. - logger.error("agent scan: failed after the model step, class=%s", type(crash).__name__) + # Logged masked, as the reading helper logs a crash. + logger.error("agent scan: failed after the model step", exc_info=crash) checked = {row.candidate.entry for row in scan.ai_rows} ai_rows = [ *scan.ai_rows, @@ -427,8 +423,8 @@ def _keep_what_was_paid_for( ) write_agents_json(output, body) return _EXIT_DONE - except Exception as failed: - logger.error("agent scan: the kept rows could not be written, class=%s", type(failed).__name__) + except Exception: + logger.error("agent scan: the kept rows could not be written", exc_info=True) try: body, _ = finish( rows=[], @@ -439,8 +435,8 @@ def _keep_what_was_paid_for( ) write_agents_json(output, body) return _EXIT_DONE - except Exception as failed: - logger.error("agent scan: the fixed error could not be written, class=%s", type(failed).__name__) + except Exception: + logger.error("agent scan: the fixed error could not be written", exc_info=True) raise crash diff --git a/src/fi/alk/harness/scan_ai.py b/src/fi/alk/harness/scan_ai.py index 1bd42174f..f539a4150 100644 --- a/src/fi/alk/harness/scan_ai.py +++ b/src/fi/alk/harness/scan_ai.py @@ -32,10 +32,12 @@ from .backends.files import SECRET_REFUSAL, above_secrets, secret_path from .config import chosen_model, load_skill from .framework_detection import _ALWAYS_SKIPPED, _TOP_ONLY_SKIPS +from .reading_entry import MaskSecrets from .scan_walk import fits_path_rule from .session import Stage, StageIdleTimeout, Turn logger = logging.getLogger(__name__) +logger.addFilter(MaskSecrets()) STAGE_NAME = "find_agents" MAX_TURNS = 40 @@ -215,6 +217,7 @@ async def _run(stage: _ScanStage, answer: _Answer, hints: str, seconds: float) - try: await asyncio.wait_for(stage.__aenter__(), OPEN_SECONDS) except Exception as exc: + logger.warning("find_agents: the model session did not open", exc_info=True) return AiOutcome( "model_error", (), @@ -243,8 +246,11 @@ def _opening(hints: str) -> str: async def _stop(stage: _ScanStage, task: asyncio.Task[Any]) -> None: try: await asyncio.wait_for(stage.interrupt_response(), INTERRUPT_SECONDS) + except asyncio.TimeoutError: + logger.warning("find_agents: the interrupt got no answer in %ss", INTERRUPT_SECONDS) except Exception: - pass + # The session may already be gone; the wait for its bill below still runs. + logger.warning("find_agents: the interrupt failed", exc_info=True) # The turn's bill arrives after the interrupt, if it arrives at all. await asyncio.wait({task}, timeout=SPEND_WAIT_SECONDS) if not task.done(): @@ -255,8 +261,11 @@ async def _stop(stage: _ScanStage, task: asyncio.Task[Any]) -> None: async def _leave(stage: _ScanStage) -> None: try: await asyncio.wait_for(stage.__aexit__(None, None, None), EXIT_SECONDS) + except asyncio.TimeoutError: + logger.warning("find_agents: the session did not close in %ss", EXIT_SECONDS) except Exception: - pass + # The answer is already kept; a session that fails to close doesn't change it. + logger.warning("find_agents: the session failed to close", exc_info=True) def _outcome( @@ -404,6 +413,8 @@ async def _answered(work: Callable[..., str], *args: Any) -> dict[str, Any]: except asyncio.TimeoutError: return _error(_TOO_SLOW) except Exception: + # The model is told only that the call failed; the trace is logged masked. + logger.warning("find_agents: a tool call failed", exc_info=True) return _error(_FAILED) @@ -423,6 +434,7 @@ async def guarded(work: Callable[[str, dict[str, Any]], str], arguments: Any) -> return _error(TOOLS_AFTER_SUBMIT) return await _answered(work, base, arguments) except Exception: + logger.warning("find_agents: a tool call failed", exc_info=True) return _error(_FAILED) # Each looks its worker up when called, not when built. @@ -933,6 +945,7 @@ async def submit_agents(arguments: dict[str, Any]) -> dict[str, Any]: except asyncio.TimeoutError: return _error(_TOO_SLOW) except Exception: + logger.warning("find_agents: a tool call failed", exc_info=True) return _error(_FAILED) return ToolServer( diff --git a/src/fi/alk/harness/scan_folders.py b/src/fi/alk/harness/scan_folders.py index 5dd781f69..148244fa5 100644 --- a/src/fi/alk/harness/scan_folders.py +++ b/src/fi/alk/harness/scan_folders.py @@ -8,6 +8,7 @@ from __future__ import annotations import logging +import traceback import multiprocessing import os import time @@ -19,14 +20,18 @@ from typing import Any from .framework_detection import detect_framework -from .source_folder import agent_folder +from .reading_entry import MaskSecrets +from .source_folder import SubfolderNotFound, agent_folder logger = logging.getLogger(__name__) +logger.addFilter(MaskSecrets()) # Loaded once in the fork server, so a restarted child doesn't import the kit again. _PRELOAD = ["fi.alk.harness.framework_detection"] # A killed child is gone almost at once; the bound only stops a wedged reap holding the scan. _JOIN_SECONDS = 5.0 +# The key of a child's answer that carries a failed check's trace instead of a framework. +_FAILED = "failed" @dataclass(frozen=True) @@ -56,9 +61,13 @@ def _answer(source: Path, subfolder: str) -> dict[str, Any]: repo_root=source, package_dir=None, ) - except Exception: + except SubfolderNotFound: # A folder the build would refuse has no word; it is not a reason to end the child. return {"name": "unknown", "version": None} + except Exception: + # A fault in the check itself. The child stays up for the next folder, and sends the + # trace so the parent logs it masked and marks the row unfinished. + return {_FAILED: traceback.format_exc()} def _serve(conn: Connection) -> None: @@ -72,6 +81,7 @@ def _serve(conn: Connection) -> None: def _framework(answer: Any) -> dict[str, Any]: + """The answer's framework, in the platform's shape; anything malformed is ``unknown``.""" name = answer.get("name") if isinstance(answer, dict) else None if not isinstance(name, str): return _unknown() @@ -135,6 +145,9 @@ def check( answer = conn.recv() except (EOFError, OSError): return self._keep(key, self._ended()) + if isinstance(answer, dict) and _FAILED in answer: + logger.warning("folder checker: the check failed in the child\n%s", answer[_FAILED]) + return self._keep(key, FolderWord(_unknown(), finished=False)) return self._keep(key, FolderWord(_framework(answer), finished=True)) def _keep(self, key: tuple[str, str], word: FolderWord) -> FolderWord: @@ -153,10 +166,7 @@ def _start(self) -> None: except Exception as error: if parent is not None: parent.close() - logger.warning( - "folder checker: the child could not start, class=%s", - type(error).__name__, - ) + logger.warning("folder checker: the child could not start", exc_info=True) raise FolderCheckerBroken( "The folder checker's child process could not start." ) from error @@ -175,7 +185,9 @@ def _ended(self) -> FolderWord: # The exit code is only known once the child is reaped. process.join(_JOIN_SECONDS) code = process.exitcode - except Exception: + except (OSError, ValueError, AssertionError): + # multiprocessing raises these for a child that is already closed or never started. + logger.warning("folder checker: the child's exit code could not be read", exc_info=True) code = None self._end_child() if code is not None and code < 0: @@ -201,13 +213,10 @@ def _end_child(self) -> None: process.kill() process.join(_JOIN_SECONDS) process.close() - except Exception as error: - logger.warning( - "folder checker: the child could not be ended, class=%s", - type(error).__name__, - ) + except Exception: + logger.warning("folder checker: the child could not be ended", exc_info=True) if conn is not None: try: conn.close() - except Exception: - pass + except OSError: + logger.debug("folder checker: the pipe was already closed", exc_info=True) diff --git a/src/fi/alk/harness/scan_walk.py b/src/fi/alk/harness/scan_walk.py index c5bdebfbf..7e7366d9f 100644 --- a/src/fi/alk/harness/scan_walk.py +++ b/src/fi/alk/harness/scan_walk.py @@ -41,9 +41,11 @@ _scan_file, _visible_tokens, ) +from .reading_entry import MaskSecrets from .source_folder import _FOLDER_FORM, _FOLDER_MAX logger = logging.getLogger(__name__) +logger.addFilter(MaskSecrets()) Sign = Literal["framework", "graph", "agent_py", "client", "model"] Hazard = Literal["", "graphs", "agent_py"] @@ -195,6 +197,19 @@ class _Unread(Exception): pass +# What reading or parsing one file can raise: that file's problem, never the walk's. Anything +# else is a fault in the kit and reaches the scan's crash path. +_FILE_ERRORS = ( + OSError, + ValueError, # UnicodeError and json's decode errors too + SyntaxError, + RecursionError, + tokenize.TokenError, + _OverBudget, + _Unread, +) + + class _FileWork: """One file's own allowance, so a heavy file can't spend another's. A count and not a clock, so the same file is read or unread every time.""" @@ -792,9 +807,14 @@ def _take(self, directory: str, folder: str, name: str) -> None: self._read_graphs(folder, path, data) else: self.templates.setdefault(folder, set()).update(_template_names(data)) + except _FILE_ERRORS: + self.unread += 1 except MemoryError: raise except Exception: + # A reader fault of a kind not foreseen is still this file's problem, never the + # walk's: the file is unread, and the trace is logged masked. + logger.warning("agent walk: a file's reader failed", exc_info=True) self.unread += 1 def _read(self, path: str) -> bytes | None: @@ -1109,17 +1129,9 @@ def walk_repo( walk.prompt_paths.clear() logger.warning("agent walk: stopped=memory unread=%d", walk.unread) return _empty_result(walk, "memory") - except Exception as exc: - # A fault outside any one file; what was read before it stands. - logger.warning("agent walk: stopped=limit class=%s", type(exc).__name__) - walk.stopped = "limit" - try: - result = walk.result(upload=upload) - except MemoryError: - raise - except Exception as exc: - logger.warning("agent walk failed: class=%s", type(exc).__name__) - result = _empty_result(walk, "limit") + # Any other fault here is the kit's, not the repository's: it goes to the scan's crash + # path, never to "the repository is larger than the scan reads". + result = walk.result(upload=upload) logger.info( "agent walk: files_read=%d unread=%d agents=%d left_out=%d stopped=%s", walk.files_read, @@ -1166,35 +1178,26 @@ def _hazard( graphs = document.get("graphs") if isinstance(document, dict) else None if isinstance(graphs, dict) and len(graphs) > 1: return "graphs" - except MemoryError: - raise - except Exception: - pass + except (OSError, ValueError, RecursionError): + # No readable langgraph.json, so it declares no second graph; the agent.py count follows. + logger.debug("hazard: langgraph.json not read", exc_info=True) # The build counts every `agent.py` below, side folders and the scan's skipped # folders included, ignoring only its own short list. found = 0 entries = 0 - try: - for index, (_, directories, files) in enumerate( - os.walk(folder, followlinks=False) - ): - # The folder's own listing is one read and always looked at; the clock bounds - # the folders below it. - if index and clock() > deadline: - return "" - directories[:] = [ - name for name in directories if name not in _BUILD_IGNORED - ] - found += files.count("agent.py") - if found > 1: - return "agent_py" - entries += len(directories) + len(files) - if entries > _HAZARD_ENTRIES: - return "" - except MemoryError: - raise - except Exception: - return "" + # os.walk reports no listing errors unless asked, so nothing here is expected to raise. + for index, (_, directories, files) in enumerate(os.walk(folder, followlinks=False)): + # The folder's own listing is one read and always looked at; the clock bounds the + # folders below it. + if index and clock() > deadline: + return "" + directories[:] = [name for name in directories if name not in _BUILD_IGNORED] + found += files.count("agent.py") + if found > 1: + return "agent_py" + entries += len(directories) + len(files) + if entries > _HAZARD_ENTRIES: + return "" return "" @@ -1240,10 +1243,14 @@ def run(self) -> None: if data is not None: try: self.templates |= _template_names(data) + except _FILE_ERRORS: + # Its names are missing, so the row's settings are short. + self.partial = True except MemoryError: raise except Exception: - pass + logger.warning("agent description: a template's reader failed", exc_info=True) + self.partial = True start = os.path.join(self.top, self.subfolder) if self.subfolder else self.top for directory, directories, files in os.walk(start, followlinks=False): directories.sort() @@ -1286,10 +1293,14 @@ def _take(self, directory: str, folder: str, name: str) -> None: self.facts[path] = _python_facts(data, self._shadowed_for(folder)) else: self.script_env |= _script_names(data) + except _FILE_ERRORS: + # Its facts are missing, so the row's counts are short. + self.partial = True except MemoryError: raise except Exception: - pass + logger.warning("agent description: a file's reader failed", exc_info=True) + self.partial = True def _read(self, path: str) -> bytes | None: """The file's bytes, or None for a file that can't or mustn't be read; sets @@ -1299,9 +1310,13 @@ def _read(self, path: str) -> bytes | None: return None try: data = _read_file(path, _FILE_BYTES + 1) - except MemoryError: - raise - except Exception: + except FileNotFoundError: + # A settings template that isn't there is the usual case, not a short count. + return None + except (OSError, ValueError): + # Unreadable, a link, or a name the system can't open: not read, so the counts + # are short. + self.partial = True return None self.files_read += 1 self.bytes_read += len(data) @@ -1347,9 +1362,10 @@ def describe_entry( reading.run() except MemoryError: raise - except Exception as exc: - # A fault outside any one file: what was read stands, and the row says so. - logger.warning("agent description stopped: class=%s", type(exc).__name__) + except Exception: + # Runs after the model was paid for, so a fault in the kit keeps what was read and + # marks the row partial rather than ending in a paid retry. The trace is logged masked. + logger.warning("agent description stopped", exc_info=True) reading.partial = True facts = reading.facts.get(entry) code = {name for found in reading.facts.values() for name in found.env} diff --git a/tests/harness/test_agent_scan.py b/tests/harness/test_agent_scan.py index b1a29f605..fba960e24 100644 --- a/tests/harness/test_agent_scan.py +++ b/tests/harness/test_agent_scan.py @@ -569,8 +569,9 @@ def test_an_upload_prefers_a_non_side_ai_row_to_a_side_walk_row(tmp_path, ai) -> assert document["truncated"] is True -def test_a_4xx_with_walk_rows_keeps_them(tmp_path, ai) -> None: - ai.outcome = _outcome("model_error", retryable=False) +@pytest.mark.parametrize("retryable", [False, True], ids=["refused", "retryable"]) +def test_a_model_error_with_walk_rows_keeps_them(tmp_path, ai, retryable) -> None: + ai.outcome = _outcome("model_error", retryable=retryable) with_rows = _write(tmp_path / "with-rows", SIDE_ONLY) assert _scan(_job(), with_rows, tmp_path / "out-rows") == 0 @@ -582,10 +583,10 @@ def test_a_4xx_with_walk_rows_keeps_them(tmp_path, ai) -> None: without = _write(tmp_path / "without", {"app/main.py": PLAIN}) - assert _scan(_job(), without, tmp_path / "out-none") == 78 + assert _scan(_job(), without, tmp_path / "out-none") == (1 if retryable else 78) failure = _read(tmp_path / "out-none", "failure.json") - assert (failure["code"], failure["retryable"]) == ("scan_model_error", False) + assert (failure["code"], failure["retryable"]) == ("scan_model_error", retryable) assert not (tmp_path / "out-none" / "agents.json").exists() @@ -811,23 +812,24 @@ def test_a_refused_pair_in_the_kept_submit_sets_truncated(tmp_path, ai) -> None: assert document["truncated"] is truncated -def test_a_crash_after_the_ai_step_logs_its_class_name_only( - tmp_path, monkeypatch, caplog, ai -) -> None: +def test_a_crash_after_the_ai_step_logs_its_trace_masked(tmp_path, monkeypatch, caplog, ai) -> None: source = _write(tmp_path / "source", SIDE_ONLY) ai.outcome = _outcome("pairs", [("app", "app/main.py")]) - sentinel = "/repo/app/.env FAKE_VALUE_123" + secret = "FAKE_SECRET_VALUE_123" + secrets = tmp_path / "secrets.json" + secrets.write_text(json.dumps({"SIMULATOR_OPENAI_API_KEY": secret}), encoding="utf-8") + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (secrets,)) def crash(*args, **kwargs): - raise RuntimeError(sentinel) + raise RuntimeError(f"bad value {secret}") monkeypatch.setattr(agent_scan, "describe_entry", crash) with caplog.at_level(logging.DEBUG, logger=agent_scan.logger.name): assert _scan(_job(), source, tmp_path / "out") == 0 - logged = [record.getMessage() for record in caplog.records if record.name == agent_scan.logger.name] - assert any("RuntimeError" in line for line in logged) - assert not any("FAKE_VALUE" in line or "/repo/" in line for line in logged) + assert "failed after the model step" in caplog.text + assert "Traceback" in caplog.text and "RuntimeError" in caplog.text + assert secret not in caplog.text assert _agents(tmp_path / "out")["status"] == "found" @@ -1172,3 +1174,22 @@ def test_an_agent_at_a_path_the_platform_cannot_carry_is_never_reported_as_none( assert _entries(document) == ["app/main.py"] assert document["truncated"] is True assert document["error"] is None + + +@pytest.mark.parametrize("step", ["run", "result"]) +def test_a_fault_in_the_walk_is_a_crash_not_a_big_repository(tmp_path, monkeypatch, ai, step) -> None: + """A bug in the walk must not be told to the person as "the repository is too large".""" + source = _write(tmp_path / "source", {"app/graph.py": GRAPH}) + + def broken(*args, **kwargs): + raise KeyError("a fault in the kit") + + monkeypatch.setattr(scan_walk._Walk, step, broken) + output = tmp_path / "out" + + assert _read_mode("github", source, output) == 1 + + failure = _read(output, "failure.json") + assert (failure["code"], failure["retryable"]) == ("scan_crashed", True) + assert not (output / "agents.json").exists() + assert len(ai.calls) == 0 diff --git a/tests/harness/test_scan_folders.py b/tests/harness/test_scan_folders.py index 91a4a2f32..0baa33a23 100644 --- a/tests/harness/test_scan_folders.py +++ b/tests/harness/test_scan_folders.py @@ -220,3 +220,18 @@ def refuse(pid: int, sig: int) -> None: real_kill(alive, signal.SIGKILL) except ProcessLookupError: pass + + +def test_a_fault_in_the_check_comes_back_as_a_trace(tmp_path, monkeypatch) -> None: + """Only a folder the build refuses is a plain ``unknown``; any other fault is sent back + with its trace, so the parent logs it and marks the row unfinished.""" + + def broken(*args, **kwargs): + raise KeyError("a fault in the check") + + monkeypatch.setattr(scan_folders, "detect_framework", broken) + answer = scan_folders._answer(tmp_path, "") + assert set(answer) == {scan_folders._FAILED} + assert "KeyError" in answer[scan_folders._FAILED] + + assert scan_folders._answer(tmp_path, "no/such/folder") == {"name": "unknown", "version": None} From 71aae2b0987d7029e8b885afc21c581064baa4fe Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 12:57:55 +0530 Subject: [PATCH 08/23] refactor(harness): use the shared reading helpers instead of copies - agent_scan imports EXIT_DONE, EXIT_RETRYABLE, limit_seconds and reading_failure from reading_entry, and EXIT_REFUSED from agent_type_check. FIND_AGENTS now sets limit_env and limit_margin_seconds, so the helper and the scan read ALK_SCAN_MAX_SECONDS at the same moment. - The find-agents stage passes idle_retries=0 itself instead of setting session.STAGE_IDLE_RETRIES for the whole process. - The walk uses framework_detection's _Work, which takes an optional limit, instead of its own copy, and bundle_author_v2's ignore list instead of a copy held equal by a test. - The walk and an AI row's description share one folder loop (_source_files) and one shadowed-name lookup (_ShadowedNames). TH-8950 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_scan.py | 63 +++++----- src/fi/alk/harness/framework_detection.py | 6 +- src/fi/alk/harness/scan_ai.py | 7 +- src/fi/alk/harness/scan_walk.py | 134 ++++++++++------------ tests/harness/test_agent_scan.py | 16 +-- tests/harness/test_scan_ai.py | 7 +- tests/harness/test_scan_walk.py | 5 - 7 files changed, 101 insertions(+), 137 deletions(-) diff --git a/src/fi/alk/harness/agent_scan.py b/src/fi/alk/harness/agent_scan.py index 0a5ada053..f574fc243 100644 --- a/src/fi/alk/harness/agent_scan.py +++ b/src/fi/alk/harness/agent_scan.py @@ -13,7 +13,6 @@ from __future__ import annotations import logging -import math import os import stat import time @@ -21,8 +20,18 @@ from dataclasses import dataclass, field, replace from pathlib import Path -from .job import FailureDomain, HarnessFailure, HarnessJob, HarnessStage, SourceKind -from .reading_entry import MaskSecrets, ReadingMode, write_reading_cost, write_reading_failure +from .job import FailureDomain, HarnessFailure, HarnessJob, SourceKind +from .agent_type_check import EXIT_REFUSED +from .reading_entry import ( + EXIT_DONE, + EXIT_RETRYABLE, + MaskSecrets, + ReadingMode, + limit_seconds, + reading_failure, + write_reading_cost, + write_reading_failure, +) from .scan_ai import AiOutcome, run_ai_step from .scan_document import Row, ScanResultInvalid, finish, root_name, write_agents_json from .scan_folders import ClockRanOut, FolderChecker, FolderCheckerBroken @@ -48,21 +57,11 @@ SCAN_FAILED_AFTER_AI = "scan_failed_after_ai" AGENT_PATH_UNSUPPORTED = "agent_path_unsupported" -_EXIT_DONE = 0 -_EXIT_RETRYABLE = 1 -_EXIT_REFUSED = 78 - - -def _seconds(name: str, default: float) -> float: - try: - value = float(os.environ.get(name, "")) - except ValueError: - return default - return value if math.isfinite(value) and value > 0 else default - - -CLOCK = _seconds("ALK_SCAN_MAX_SECONDS", 1200.0) -AI_SECONDS = _seconds("ALK_SCAN_AI_MAX_SECONDS", 600.0) +CLOCK_ENV = "ALK_SCAN_MAX_SECONDS" +# The scan's clock when CLOCK_ENV doesn't name one. The helper reads the same variable for its +# hard stop, both when the reading starts. +CLOCK = 1200.0 +AI_SECONDS = limit_seconds("ALK_SCAN_AI_MAX_SECONDS", 600.0) # The helper's hard stop comes this long after the scan's own clock, so a scan that stops itself # always writes its own file first. HARD_LIMIT_MARGIN = 600.0 @@ -117,28 +116,18 @@ class _Scan: models_first: bool = False -def _failure(domain: FailureDomain, code: str, message: str, *, retryable: bool) -> HarnessFailure: - return HarnessFailure( - domain=domain, - stage=HarnessStage.UNDERSTANDING_AGENT, - code=code, - message=message, - retryable=retryable, - ) - - def _fail( output: Path, domain: FailureDomain, code: str, message: str, *, retryable: bool ) -> int: - write_reading_failure(_failure(domain, code, message, retryable=retryable), output) + write_reading_failure(reading_failure(domain, code, message, retryable=retryable), output) logger.warning("agent scan: failed, code=%s retryable=%s", code, retryable) - return _EXIT_RETRYABLE if retryable else _EXIT_REFUSED + return EXIT_RETRYABLE if retryable else EXIT_REFUSED def _check_job(job: HarnessJob) -> HarnessFailure | None: source = job.source if source.kind in (SourceKind.REMOTE, SourceKind.PROVIDER) or source.subfolder: - return _failure( + return reading_failure( FailureDomain.PLATFORM_SYNC, SCAN_JOB_INVALID, "The agent scan reads a whole GitHub repository or an upload; this job names a " @@ -273,7 +262,7 @@ def _write( truncated, error or "none", ) - return _EXIT_DONE + return EXIT_DONE def _result( @@ -422,7 +411,7 @@ def _keep_what_was_paid_for( models_first=scan.models_first, ) write_agents_json(output, body) - return _EXIT_DONE + return EXIT_DONE except Exception: logger.error("agent scan: the kept rows could not be written", exc_info=True) try: @@ -434,7 +423,7 @@ def _keep_what_was_paid_for( error=(SCAN_FAILED_AFTER_AI, _ERROR_MESSAGES[SCAN_FAILED_AFTER_AI]), ) write_agents_json(output, body) - return _EXIT_DONE + return EXIT_DONE except Exception: logger.error("agent scan: the fixed error could not be written", exc_info=True) raise crash @@ -443,7 +432,7 @@ def _keep_what_was_paid_for( async def _scan(job: HarnessJob, source: Path, output: Path, scan: _Scan) -> int: clock = time.monotonic # Taken first, so the source check counts against the clock too. - deadline = clock() + CLOCK + deadline = clock() + limit_seconds(CLOCK_ENV, CLOCK) if not _holds_a_file(source): return _fail( output, @@ -536,7 +525,9 @@ async def _work(job: HarnessJob, source: Path, output: Path) -> int: name="find_agents", check_job=_check_job, work=_work, - time_limit_seconds=CLOCK + HARD_LIMIT_MARGIN, + time_limit_seconds=CLOCK, + limit_env=CLOCK_ENV, + limit_margin_seconds=HARD_LIMIT_MARGIN, timed_out_code=SCAN_HARD_LIMIT_REACHED, timed_out_retryable=False, crashed_code=SCAN_CRASHED, diff --git a/src/fi/alk/harness/framework_detection.py b/src/fi/alk/harness/framework_detection.py index 8c6705b0f..6b90fe5a6 100644 --- a/src/fi/alk/harness/framework_detection.py +++ b/src/fi/alk/harness/framework_detection.py @@ -284,12 +284,14 @@ class _Work: every time. Python 3.12.3 re-decodes a token's whole line for each token, and one name can have thousands of targets: neither shows in the size of a file.""" - def __init__(self) -> None: + def __init__(self, limit: int | None = None) -> None: self._spent = 0 + # None follows _MAX_WORK; the agent scan gives each file its own, smaller limit. + self._limit = limit def charge(self, amount: int) -> None: self._spent += amount - if self._spent > _MAX_WORK: + if self._spent > (_MAX_WORK if self._limit is None else self._limit): raise _OverBudget() diff --git a/src/fi/alk/harness/scan_ai.py b/src/fi/alk/harness/scan_ai.py index f539a4150..5e3874b71 100644 --- a/src/fi/alk/harness/scan_ai.py +++ b/src/fi/alk/harness/scan_ai.py @@ -27,7 +27,6 @@ from pathlib import Path from typing import Any, Literal -from . import session from .backends import SessionSpec, StageDone, ToolServer, ToolSpec from .backends.files import SECRET_REFUSAL, above_secrets, secret_path from .config import chosen_model, load_skill @@ -151,7 +150,9 @@ class _ScanStage(Stage): def __init__( self, spec: SessionSpec, *, name: str, on_turn_end: Callable[[], None] ) -> None: - super().__init__(spec, name=name) + # A replayed quiet stream would be a second paid opening, and its first spend is + # never recorded. + super().__init__(spec, name=name, idle_retries=0) self.done: list[StageDone] = [] self._on_turn_end = on_turn_end @@ -180,8 +181,6 @@ async def run_ai_step( source: Path, *, hints: str, seconds: float, on_turn_end: Callable[[], None] ) -> AiOutcome: """Ask the model once for the repository's agents, within ``seconds``.""" - # A replayed quiet stream is a second paid opening. - session.STAGE_IDLE_RETRIES = 0 answer = _Answer() spec = SessionSpec( system_prompt=load_skill("find-agents"), diff --git a/src/fi/alk/harness/scan_walk.py b/src/fi/alk/harness/scan_walk.py index 7e7366d9f..366acf610 100644 --- a/src/fi/alk/harness/scan_walk.py +++ b/src/fi/alk/harness/scan_walk.py @@ -40,7 +40,9 @@ _read_file, _scan_file, _visible_tokens, + _Work, ) +from .bundle_author_v2 import _IGNORED_ARTIFACT_PARTS as _BUILD_IGNORED from .reading_entry import MaskSecrets from .source_folder import _FOLDER_FORM, _FOLDER_MAX @@ -86,9 +88,6 @@ _BUILT_FOLDERS = frozenset({"dist", "build", "static", "vendor", "node_modules"}) _BUILT_ENDINGS = (".min.js", ".d.ts", ".d.mts", ".d.cts") _CONFIG_ENDINGS = tuple(".config" + suffix for suffix in _SCRIPT_SUFFIXES) -# What a build's fallback ignores when it counts `agent.py`. A copy, because the build's -# module is heavy to import; a test holds the two equal. -_BUILD_IGNORED = frozenset({".git", ".venv", "__pycache__", "node_modules"}) _CLIENTS = { ("OpenAI",): "openai", @@ -210,19 +209,6 @@ class _Unread(Exception): ) -class _FileWork: - """One file's own allowance, so a heavy file can't spend another's. A count and not - a clock, so the same file is read or unread every time.""" - - def __init__(self) -> None: - self._spent = 0 - - def charge(self, amount: int) -> None: - self._spent += amount - if self._spent > _FILE_WORK: - raise _OverBudget() - - def fits_path_rule(path: str) -> bool: """Whether the platform can carry a non-empty path. The root, ``""``, isn't one.""" return ( @@ -459,15 +445,15 @@ def _python_facts(data: bytes, shadowed: frozenset[str]) -> _Facts: """Raises for a file that has to count as unread.""" if _has_long_line(data): raise _Unread() - work = _FileWork() - scan = _FileScan(shadowed, frozenset(), work) # type: ignore[arg-type] + work = _Work(_FILE_WORK) + scan = _FileScan(shadowed, frozenset(), work) if _scan_file(data, scan, read_uses=True): raise _Unread() own = _OwnReader() with warnings.catch_warnings(): # On 3.12 `tokenize` and `literal_eval` warn once per `\{`, which floods stderr. warnings.simplefilter("ignore") - own.read(_visible_tokens(data, work)) # type: ignore[arg-type] + own.read(_visible_tokens(data, work)) builds = sorted( (framework, line) for (framework, strength), line in scan.found.items() @@ -727,6 +713,42 @@ def _components(nodes: list[str], edges: dict[str, list[str]]) -> list[list[str] return components +def _source_files(top: str, start: str) -> Iterator[tuple[str, str, str]]: + """Every file below ``start`` the scan may read, as (directory, folder relative to + ``top``, name), in sorted order so that a walk stopped by a limit stops at the same file. + Installed trees and the folders a build skips are never entered.""" + for directory, directories, files in os.walk(start, followlinks=False): + directories.sort() + files.sort() + relative = directory[len(top) :].split(os.sep)[1:] + if relative and any(name.endswith(".dist-info") for name in directories): + # An installed tree, not the customer's code. + directories[:] = [] + continue + directories[:] = _folders_to_enter(directory, directories, at_top=not relative) + folder = "/".join(relative) + for name in files: + yield directory, folder, name + + +class _ShadowedNames: + """Each project's own module names, read once per project, so an import of the + customer's own ``agents`` package isn't read as a framework.""" + + def __init__(self, top: str, projects: _Projects) -> None: + self._top = top + self._projects = projects + self._by_project: dict[str, frozenset[str]] = {} + + def for_folder(self, folder: str) -> frozenset[str]: + project = self._projects.nearest(folder) or "" + if project not in self._by_project: + self._by_project[project] = _own_modules( + Path(self._top, project) if project else Path(self._top), set() + ) + return self._by_project[project] + + class _Walk: """One walk of the source. Holds every file's facts until the agents are decided.""" @@ -754,31 +776,18 @@ def __init__( self.source_outside_side = False self.script_outside_side = False self.projects = _Projects(self.top, _PROJECT_FILES) - self._shadowed: dict[str, frozenset[str]] = {} + self.shadowed = _ShadowedNames(self.top, self.projects) self._roots: dict[str, list[str]] = {} self._edges: dict[str, tuple[str, ...]] = {} def run(self) -> None: - for directory, directories, files in os.walk(self.top, followlinks=False): - # Sorted, so that a walk stopped by a limit stops at the same file. - directories.sort() - files.sort() - relative = directory[len(self.top) :].split(os.sep)[1:] - if relative and any(name.endswith(".dist-info") for name in directories): - # An installed tree, not the customer's code. - directories[:] = [] - continue - directories[:] = _folders_to_enter( - directory, directories, at_top=not relative - ) - folder = "/".join(relative) - for name in files: - if self.clock() > self.deadline: - self.stopped = "clock" - return - self._take(directory, folder, name) - if self.stopped: - return + for directory, folder, name in _source_files(self.top, self.top): + if self.clock() > self.deadline: + self.stopped = "clock" + return + self._take(directory, folder, name) + if self.stopped: + return def _take(self, directory: str, folder: str, name: str) -> None: path = _join(folder, name) @@ -802,7 +811,7 @@ def _take(self, directory: str, folder: str, name: str) -> None: if data is None: return if name.endswith(".py"): - self.facts[path] = _python_facts(data, self._shadowed_for(folder)) + self.facts[path] = _python_facts(data, self.shadowed.for_folder(folder)) elif name == _LANGGRAPH: self._read_graphs(folder, path, data) else: @@ -846,14 +855,6 @@ def _read_graphs(self, folder: str, path: str, data: bytes) -> None: if target not in (".", "..") and not target.startswith(("../", "/")): self.graphs.setdefault(target, path) - def _shadowed_for(self, folder: str) -> frozenset[str]: - project = self.projects.nearest(folder) or "" - if project not in self._shadowed: - self._shadowed[project] = _own_modules( - Path(self.top, project) if project else Path(self.top), set() - ) - return self._shadowed[project] - def _import_roots(self, folder: str) -> list[str]: """Where an absolute import from ``folder`` is looked for, nearest first.""" if folder not in self._roots: @@ -1226,7 +1227,7 @@ def __init__( self.files_read = 0 self.bytes_read = 0 self.partial = False - self._shadowed: dict[str, frozenset[str]] = {} + self.shadowed = _ShadowedNames(top, self.projects) def run(self) -> None: if _has_link(self.top, self.subfolder): @@ -1252,23 +1253,12 @@ def run(self) -> None: logger.warning("agent description: a template's reader failed", exc_info=True) self.partial = True start = os.path.join(self.top, self.subfolder) if self.subfolder else self.top - for directory, directories, files in os.walk(start, followlinks=False): - directories.sort() - files.sort() - relative = directory[len(self.top) :].split(os.sep)[1:] - if relative and any(name.endswith(".dist-info") for name in directories): - directories[:] = [] - continue - directories[:] = _folders_to_enter( - directory, directories, at_top=not relative - ) - folder = "/".join(relative) - for name in files: - if self._out_of_time(): - return - self._take(directory, folder, name) - if self.partial: - return + for directory, folder, name in _source_files(self.top, start): + if self._out_of_time(): + return + self._take(directory, folder, name) + if self.partial: + return def _out_of_time(self) -> bool: if self.clock() > self.deadline: @@ -1290,7 +1280,7 @@ def _take(self, directory: str, folder: str, name: str) -> None: return try: if name.endswith(".py"): - self.facts[path] = _python_facts(data, self._shadowed_for(folder)) + self.facts[path] = _python_facts(data, self.shadowed.for_folder(folder)) else: self.script_env |= _script_names(data) except _FILE_ERRORS: @@ -1325,14 +1315,6 @@ def _read(self, path: str) -> bytes | None: return None return data if len(data) <= _FILE_BYTES else None - def _shadowed_for(self, folder: str) -> frozenset[str]: - project = self.projects.nearest(folder) or "" - if project not in self._shadowed: - self._shadowed[project] = _own_modules( - Path(self.top, project) if project else Path(self.top), set() - ) - return self._shadowed[project] - def describe_entry( source: Path, diff --git a/tests/harness/test_agent_scan.py b/tests/harness/test_agent_scan.py index fba960e24..31a0ba2cb 100644 --- a/tests/harness/test_agent_scan.py +++ b/tests/harness/test_agent_scan.py @@ -266,22 +266,14 @@ def test_the_mode_settings(monkeypatch) -> None: assert FIND_AGENTS.check_job is _check_job assert FIND_AGENTS.work is _work assert inspect.iscoroutinefunction(FIND_AGENTS.work) - # The environment hasn't changed since import, so this is the clock the mode was built with. - clock = agent_scan._seconds("ALK_SCAN_MAX_SECONDS", 1200.0) - assert FIND_AGENTS.time_limit_seconds == clock + 600 + # The helper reads the scan's own clock variable for its hard stop, as the scan does. + assert FIND_AGENTS.time_limit_seconds == agent_scan.CLOCK == 1200.0 + assert FIND_AGENTS.limit_env == agent_scan.CLOCK_ENV == "ALK_SCAN_MAX_SECONDS" + assert FIND_AGENTS.limit_margin_seconds == 600.0 assert FIND_AGENTS.timed_out_code == "scan_hard_limit_reached" assert FIND_AGENTS.timed_out_retryable is False assert FIND_AGENTS.crashed_code == "scan_crashed" - for name, default in (("ALK_SCAN_MAX_SECONDS", 1200.0), ("ALK_SCAN_AI_MAX_SECONDS", 600.0)): - monkeypatch.delenv(name, raising=False) - assert agent_scan._seconds(name, default) == default - for raw in ("", "abc", "-5", "0", "inf", "nan"): - monkeypatch.setenv(name, raw) - assert agent_scan._seconds(name, default) == default - monkeypatch.setenv(name, "900") - assert agent_scan._seconds(name, default) == 900.0 - def test_a_missing_or_empty_source_is_a_retryable_failure(tmp_path, ai) -> None: a_file = _write(tmp_path / "files", {"bot.py": GRAPH}) / "bot.py" diff --git a/tests/harness/test_scan_ai.py b/tests/harness/test_scan_ai.py index be3cca7bd..5421dd15b 100644 --- a/tests/harness/test_scan_ai.py +++ b/tests/harness/test_scan_ai.py @@ -324,6 +324,7 @@ def test_each_ai_outcome_gives_its_result(tmp_path, monkeypatch, quick, name): case = _OUTCOMES[name] if case.idle is not None: monkeypatch.setattr(scan_ai, "IDLE_SECONDS", case.idle) + session.STAGE_IDLE_RETRIES = 1 backend = _Backend(*case.steps, interrupt=case.interrupt) ends: list = [] @@ -334,7 +335,8 @@ def test_each_ai_outcome_gives_its_result(tmp_path, monkeypatch, quick, name): assert outcome.complete is case.complete assert outcome.retryable is case.retryable assert len(ends) == case.ends - assert session.STAGE_IDLE_RETRIES == 0 + # The scan's stage turns replay off for itself, never for the whole process. + assert session.STAGE_IDLE_RETRIES == 1 assert len(backend.sent) == 1 if case.kind == "timed_out": assert backend.interrupts == 1 @@ -417,7 +419,8 @@ def test_an_idle_ai_stream_is_not_replayed(tmp_path, monkeypatch, quick): assert outcome.retryable is True assert len(backend.specs) == 1 assert len(backend.sent) == 1 - assert session.STAGE_IDLE_RETRIES == 0 + # The scan's stage turns replay off for itself, never for the whole process. + assert session.STAGE_IDLE_RETRIES == 1 def test_pairs_submitted_before_an_idle_stream_are_kept(tmp_path, monkeypatch, quick): diff --git a/tests/harness/test_scan_walk.py b/tests/harness/test_scan_walk.py index 243c0df72..2de3c642f 100644 --- a/tests/harness/test_scan_walk.py +++ b/tests/harness/test_scan_walk.py @@ -140,11 +140,6 @@ def test_the_private_detector_names_it_uses_exist() -> None: "work", ] assert "read_uses" in inspect.signature(fd._scan_file).parameters - from fi.alk.harness import bundle_author_v2 - - assert scan_walk._BUILD_IGNORED == frozenset( - bundle_author_v2._IGNORED_ARTIFACT_PARTS - ) @pytest.mark.parametrize( From 2b6b816c50779b86040aed55bc137de6c36487a9 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 12:59:32 +0530 Subject: [PATCH 09/23] fix(harness): keep an agent's folder stable, and read own code first - An agent's subfolder counts only the agents of its own project. Adding an unrelated project with its own agent deeper in the repo no longer moves the first agent's folder, which the platform keys on. - The walk reads files outside test, example and other side folders first, then the side folders. A large tests folder sorted ahead of the agent can no longer use up the 20,000-file cap before the agent is read. TH-8950 Signed-off-by: khushalsonawat --- src/fi/alk/harness/scan_walk.py | 33 ++++++++++++++++++++++----------- tests/harness/test_scan_walk.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/src/fi/alk/harness/scan_walk.py b/src/fi/alk/harness/scan_walk.py index 366acf610..90a656357 100644 --- a/src/fi/alk/harness/scan_walk.py +++ b/src/fi/alk/harness/scan_walk.py @@ -713,10 +713,13 @@ def _components(nodes: list[str], edges: dict[str, list[str]]) -> list[list[str] return components -def _source_files(top: str, start: str) -> Iterator[tuple[str, str, str]]: +def _source_files( + top: str, start: str, *, side: bool | None = None +) -> Iterator[tuple[str, str, str]]: """Every file below ``start`` the scan may read, as (directory, folder relative to ``top``, name), in sorted order so that a walk stopped by a limit stops at the same file. - Installed trees and the folders a build skips are never entered.""" + Installed trees and the folders a build skips are never entered. ``side`` False gives only + files outside test, example and other side folders; True gives only the files in them.""" for directory, directories, files in os.walk(start, followlinks=False): directories.sort() files.sort() @@ -726,9 +729,12 @@ def _source_files(top: str, start: str) -> Iterator[tuple[str, str, str]]: directories[:] = [] continue directories[:] = _folders_to_enter(directory, directories, at_top=not relative) + if side is False: + directories[:] = [name for name in directories if not _is_side_part(name)] folder = "/".join(relative) for name in files: - yield directory, folder, name + if side is None or _is_side_path(_join(folder, name)) is side: + yield directory, folder, name class _ShadowedNames: @@ -781,13 +787,16 @@ def __init__( self._edges: dict[str, tuple[str, ...]] = {} def run(self) -> None: - for directory, folder, name in _source_files(self.top, self.top): - if self.clock() > self.deadline: - self.stopped = "clock" - return - self._take(directory, folder, name) - if self.stopped: - return + # The customer's own code first, then side folders, so tests and examples can't use + # up the file cap before the agent is reached. + for side in (False, True): + for directory, folder, name in _source_files(self.top, self.top, side=side): + if self.clock() > self.deadline: + self.stopped = "clock" + return + self._take(directory, folder, name) + if self.stopped: + return def _take(self, directory: str, folder: str, name: str) -> None: path = _join(folder, name) @@ -986,10 +995,12 @@ def _subfolder(self, entry: str, side: bool, agents: list[tuple[str, bool]]) -> project = self.projects.nearest(folder) if project is None: return folder + # Only agents of this same project count: one in a deeper project of its own + # doesn't move this agent's folder. sharing = sum( 1 for other, other_side in agents - if not other_side and _under(other, project) + if not other_side and self._project_of(other) == project ) return project if sharing == 1 else folder diff --git a/tests/harness/test_scan_walk.py b/tests/harness/test_scan_walk.py index 2de3c642f..80b4824ba 100644 --- a/tests/harness/test_scan_walk.py +++ b/tests/harness/test_scan_walk.py @@ -1194,3 +1194,34 @@ def test_a_chat_model_beside_a_stronger_agent_is_not_listed(tmp_path: Path) -> N "svc/graph.py": "framework", "other/chain.py": "model", } + + +def test_a_deeper_project_does_not_move_an_agents_folder(tmp_path: Path) -> None: + """The platform keys an agent on repo and subfolder, so adding an unrelated project with + its own agent must not move the first agent's folder.""" + project = "[project]\nname = 'x'\n" + _write(tmp_path, {"pyproject.toml": project, "src/app/agent.py": GRAPH}) + alone = _walk(tmp_path) + assert {agent.entry: agent.subfolder for agent in alone.agents} == {"src/app/agent.py": ""} + + _write(tmp_path, {"plugins/x/pyproject.toml": project, "plugins/x/agent.py": GRAPH}) + beside = _walk(tmp_path) + assert {agent.entry: agent.subfolder for agent in beside.agents} == { + "src/app/agent.py": "", + "plugins/x/agent.py": "plugins/x", + } + + +def test_side_folders_cannot_use_up_the_file_cap_before_the_agent( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The customer's own code is read first; a big tests folder sorted ahead of the agent + no longer hides it behind the file cap.""" + monkeypatch.setattr(scan_walk, "_MAX_FILES", 10) + _write(tmp_path, {f"tests/test_{index:02}.py": "x = 1\n" for index in range(25)}) + _write(tmp_path, {"zz_agent/agent.py": GRAPH}) + + result = _walk(tmp_path) + + assert "zz_agent/agent.py" in _entries(result) + assert result.stopped == "limit" From 67f682ad625407000172589a62dca434d9ee6911 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 13:00:38 +0530 Subject: [PATCH 10/23] docs(harness): say what a build can do with a shared folder, and name tests plainly - The shared-folder evidence sentence no longer asks a build to name an entry, which a build job has no field for. - Test names say "the platform format", "the platform rules" and "the path rule" instead of "his". - _result's docstring describes what it does instead of pointing at plan step numbers. TH-8950 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_scan.py | 3 ++- src/fi/alk/harness/scan_document.py | 2 +- tests/harness/test_scan_document.py | 10 +++++----- tests/harness/test_scan_walk.py | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/fi/alk/harness/agent_scan.py b/src/fi/alk/harness/agent_scan.py index f574fc243..4278a8480 100644 --- a/src/fi/alk/harness/agent_scan.py +++ b/src/fi/alk/harness/agent_scan.py @@ -272,7 +272,8 @@ def _result( walk: WalkResult, outcome: AiOutcome | None, ) -> int: - """Steps 6.4 to 9: the one row of an upload, the model's outcome, then the file.""" + """Turn the rows and the model's outcome into the file: an upload keeps one row, the + outcome decides the status and ``truncated``, and the walk's flags add to ``truncated``.""" if scan.upload: rows, others = _one_row(scan.walk_rows, scan.ai_rows) else: diff --git a/src/fi/alk/harness/scan_document.py b/src/fi/alk/harness/scan_document.py index 3ea15d0e2..961a989b5 100644 --- a/src/fi/alk/harness/scan_document.py +++ b/src/fi/alk/harness/scan_document.py @@ -102,7 +102,7 @@ "Uses the OpenAI Agents SDK, set aside because a folder named agents/ shadows it." ), "shares_folder": ( - "Another agent listed here shares this folder, so a build must name its entry." + "Another agent here shares this folder; for now only the folder can be picked." ), "graphs_may_stop": ( "This folder's langgraph.json lists more than one graph, so a build may stop." diff --git a/tests/harness/test_scan_document.py b/tests/harness/test_scan_document.py index 0c8efe645..f44aa263e 100644 --- a/tests/harness/test_scan_document.py +++ b/tests/harness/test_scan_document.py @@ -87,7 +87,7 @@ def _confidence(*rows: Row) -> list[str]: return [agent["confidence"] for agent in document["agents"]] -def test_found_lists_each_agent_in_his_format() -> None: +def test_found_lists_each_agent_in_the_platform_format() -> None: graph = _row( "services/support/graph.py", "services/support", @@ -254,7 +254,7 @@ def test_rows_and_an_error_together_keep_the_rows() -> None: assert dropped is True -def test_names_evidence_and_paths_always_fit_his_rules() -> None: +def test_names_evidence_and_paths_always_fit_the_platform_rules() -> None: def job(agent_name: Any = None, repository: Any = None) -> Any: metadata = {} if agent_name is None else {"agent_name": agent_name} return SimpleNamespace(metadata=metadata, source=SimpleNamespace(repository=repository)) @@ -334,7 +334,7 @@ def test_clean_name_collapses_every_kind_of_space() -> None: ) -def test_a_sentence_never_names_a_path_his_rule_refuses() -> None: +def test_a_sentence_never_names_a_path_the_path_rule_refuses() -> None: undecodable = os.fsdecode(b"agent\xff.py") refused = [ undecodable, @@ -457,7 +457,7 @@ def entries(chosen: list[Row]) -> list[str]: assert written == ["a/graph.py", *(f"h{i:02}/main.py" for i in range(48)), "z/agent.ts"] -def test_a_row_his_rules_reject_is_dropped_with_truncated(caplog: pytest.LogCaptureFixture) -> None: +def test_a_row_the_platform_rules_reject_is_dropped_with_truncated(caplog: pytest.LogCaptureFixture) -> None: good = _row("good/main.py", "good") bad = [ _row("bad/../escape.py", "bad"), @@ -519,7 +519,7 @@ def problem(**changes: Any) -> str: ] -def test_the_file_matches_his_serializer() -> None: +def test_the_file_matches_the_platform_serializer() -> None: fixture = json.loads(FIXTURE.read_text(encoding="utf-8")) assert fixture["serializer_commit"] == "e07d56fcd" cases = fixture["cases"] diff --git a/tests/harness/test_scan_walk.py b/tests/harness/test_scan_walk.py index 80b4824ba..cd92ea374 100644 --- a/tests/harness/test_scan_walk.py +++ b/tests/harness/test_scan_walk.py @@ -596,7 +596,7 @@ def test_more_than_fifty_agents_in_one_folder_keeps_fifty(tmp_path: Path) -> Non assert not exact.more_than_fifty -def test_paths_his_rule_rejects_are_left_out(tmp_path: Path) -> None: +def test_paths_the_path_rule_rejects_are_left_out(tmp_path: Path) -> None: long_folder = "/".join(["d" * 100] * 5) _write( tmp_path, From 4617eecff470406011f8c840d30363aebe2561e5 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 13:04:27 +0530 Subject: [PATCH 11/23] fix(harness): keep a saved reading through the hard limit and a cancel - At the hard limit a saved contract becomes agent.json and exit 0, instead of a timeout written over it; the close after a stop is bounded. - A cancel is a crash with its failure before the save, and keeps the saved contract after it. - The build and the reading share one agent-document helper, so a folder mismatch reports the build's code. - A failed document logs a masked trace; provider-import and connect-only jobs are refused; a soft-limit test uses a fake clock. TH-8951 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_reading.py | 40 +++++-- src/fi/alk/harness/reading_entry.py | 61 +++++++--- src/fi/alk/harness/reading_result.py | 35 +++--- src/fi/alk/harness/scenario_source.py | 41 ++++--- tests/harness/test_agent_reading.py | 161 +++++++++++++++++++++++++- tests/harness/test_reading_entry.py | 49 +++++++- 6 files changed, 322 insertions(+), 65 deletions(-) diff --git a/src/fi/alk/harness/agent_reading.py b/src/fi/alk/harness/agent_reading.py index ee017adf6..eb5f6b29b 100644 --- a/src/fi/alk/harness/agent_reading.py +++ b/src/fi/alk/harness/agent_reading.py @@ -22,8 +22,9 @@ from .backends import StageDone from .cli import guidance_text from .failure_file import FAILURE_FILE -from .job import FailureDomain, HarnessFailure, HarnessJob, SourceKind +from .job import FailureDomain, HarnessFailure, HarnessJob, ProviderExecutionMode, SourceKind from .reading_entry import ( + EXIT_DONE, EXIT_RETRYABLE, READING_JOB_INVALID, MaskSecrets, @@ -34,6 +35,7 @@ write_reading_failure, ) from .reading_result import ( + AGENT_FILE, CONTRACT_FILE, classify, document, @@ -70,6 +72,15 @@ def _soft_limit() -> float: def _check_job(job: HarnessJob) -> HarnessFailure | None: + if job.agent.mode in (ProviderExecutionMode.PROVIDER_IMPORT, ProviderExecutionMode.CONNECT_ONLY): + # The build reads these agents from the provider's definition, which a reading never loads. + return reading_failure( + FailureDomain.PLATFORM_SYNC, + READING_JOB_INVALID, + "A reading without a build reads an agent from its repository only. A " + f"{job.agent.mode.value} job's agent is defined by its provider, so it needs a build.", + retryable=False, + ) if job.source.kind in (SourceKind.GITHUB, SourceKind.ARCHIVE): return None return reading_failure( @@ -176,14 +187,14 @@ async def _stop( # A stop that can't be delivered is still a timeout, not a crash. logger.warning("the stop at the time limit was not delivered", exc_info=True) await asyncio.wait({task}, timeout=wait) - # Closed before the files are read, so no late tool call can save after the decision. + # Closed before the files are read, so no late tool call can save after the decision. The + # close is bounded too: a session that hangs closing must not keep the reading from deciding. task.cancel() - try: - await task - except asyncio.CancelledError: - pass - except Exception: - logger.warning("the stopped reading ended with an error", exc_info=True) + await asyncio.wait({task}, timeout=wait) + if not task.done(): + logger.warning("the stopped reading's session did not close within %g s", wait) + elif not task.cancelled() and task.exception() is not None: + logger.warning("the stopped reading ended with an error", exc_info=task.exception()) def _timed_out(output: Path, soft_limit: float) -> int: @@ -297,8 +308,9 @@ async def _read(job: HarnessJob, source: Path, output: Path, *, interrupt_wait: code, complete = await _reading( job, Path(source), output, started, _soft_limit(), interrupt_wait ) - except Exception as exc: + except (Exception, asyncio.CancelledError) as exc: # A contract the gate saved is paid for; a crash would have the platform pay again. + # A cancel leaking out of the session is the same: the saved contract still stands. if saved_file(output) != CONTRACT_FILE: raise logger.warning( @@ -311,6 +323,15 @@ async def _read(job: HarnessJob, source: Path, output: Path, *, interrupt_wait: return code +def _at_the_limit(job: HarnessJob, source: Path, output: Path) -> int | None: + """At the helper's hard limit: a saved contract still ends as the reading ends one.""" + if saved_file(output) != CONTRACT_FILE: + return None + if (output / AGENT_FILE).is_file(): + return EXIT_DONE + return document(job, Path(source), output) + + def _read_agent(*, interrupt_wait: float = _INTERRUPT_WAIT_SECONDS) -> ReadingMode: return ReadingMode( name="read_agent", @@ -324,6 +345,7 @@ def _read_agent(*, interrupt_wait: float = _INTERRUPT_WAIT_SECONDS) -> ReadingMo crashed_code=AGENT_READING_CRASHED, limit_env=_LIMIT_ENV, limit_margin_seconds=_HARD_LIMIT_MARGIN_SECONDS, + at_limit=_at_the_limit, ) diff --git a/src/fi/alk/harness/reading_entry.py b/src/fi/alk/harness/reading_entry.py index 400f1578b..9a0dd66c4 100644 --- a/src/fi/alk/harness/reading_entry.py +++ b/src/fi/alk/harness/reading_entry.py @@ -38,6 +38,8 @@ EXIT_DONE = 0 EXIT_RETRYABLE = 1 +# How long a mode's own end at the hard limit may take before the timeout is written anyway. +_SETTLE_SECONDS = 60.0 # Neither file is deleted by the hosted authoring entrypoint, so a read here takes nothing away # from the code that loads them. @@ -64,6 +66,9 @@ class ReadingMode: # ``time_limit_seconds`` as its default; the helper's own limit adds the margin to it. limit_env: str | None = None limit_margin_seconds: float = 0.0 + # Called at the hard limit, before any timeout is written: an exit code means the mode has + # already ended what it saved (its own files written), and no timeout is written over it. + at_limit: Callable[[HarnessJob, Path, Path], int | None] | None = None def limit_seconds(env: str, default: float) -> float: @@ -238,8 +243,12 @@ class _HardLimit: """Ends the process at the limit. Before the work returns, a timeout is written first; after it, the work's files stand and the process ends with the work's own code.""" - def __init__(self, mode: ReadingMode, output: Path, limit: float) -> None: + def __init__( + self, mode: ReadingMode, job: HarnessJob, source: Path, output: Path, limit: float + ) -> None: self._mode = mode + self._job = job + self._source = source self._output = output self._limit = limit self._returned: int | None = None @@ -264,21 +273,44 @@ def _end_the_process(self) -> None: # Nothing logs on the way out: a blocked stderr must not hold off the exit. logging.disable(logging.CRITICAL) try: - write_reading_failure( - reading_failure( - FailureDomain.INFRASTRUCTURE, - mode.timed_out_code, - f"The {mode.name} reading ran past its {limit:g} s limit " - "and was stopped.", - retryable=mode.timed_out_retryable, - ), - output, - ) + settled = self._settled() + if settled is not None: + # The mode ended what it had saved; a timeout would throw away a paid result. + exit_code = settled + else: + write_reading_failure( + reading_failure( + FailureDomain.INFRASTRUCTURE, + mode.timed_out_code, + f"The {mode.name} reading ran past its {limit:g} s limit " + "and was stopped.", + retryable=mode.timed_out_retryable, + ), + output, + ) write_reading_cost(output, complete=False) finally: # Work running in a thread can't be cancelled, and a normal exit would wait for it. os._exit(exit_code) + def _settled(self) -> int | None: + at_limit = self._mode.at_limit + if at_limit is None: + return None + settled: list[int | None] = [] + + def settle() -> None: + try: + settled.append(at_limit(self._job, self._source, self._output)) + except Exception: + pass + + # Its own thread, with a bound, so the mode's last step can't keep the process alive. + worker = threading.Thread(target=settle, daemon=True) + worker.start() + worker.join(_SETTLE_SECONDS) + return settled[0] if settled else None + async def _work_then_stop_the_clock( mode: ReadingMode, job: HarnessJob, source: Path, output: Path, limit: _HardLimit | None @@ -360,12 +392,13 @@ def run_reading_mode(mode: ReadingMode, *, job_path: Path, source: Path, output: # Read now, not when the mode was made, so it is the same limit the work reads as it starts. seconds = _hard_limit(mode) - limit = None if seconds is None else _HardLimit(mode, output, seconds) - crash: Exception | None = None + limit = None if seconds is None else _HardLimit(mode, job, source, output, seconds) + crash: BaseException | None = None try: logger.info("reading started at %s", datetime.now(timezone.utc).isoformat()) code = asyncio.run(_work_then_stop_the_clock(mode, job, source, output, limit)) - except Exception as exc: + except (Exception, asyncio.CancelledError) as exc: + # A cancel leaking out of the work is a crash too; an interrupt or exit is let through. logger.error("the %s reading crashed", mode.name, exc_info=True) crash = exc finally: diff --git a/src/fi/alk/harness/reading_result.py b/src/fi/alk/harness/reading_result.py index 1b5dab92e..9f92fc9bd 100644 --- a/src/fi/alk/harness/reading_result.py +++ b/src/fi/alk/harness/reading_result.py @@ -14,20 +14,25 @@ from pathlib import Path from typing import TYPE_CHECKING, Any -from .. import __version__ -from ..providers.agent import AgentBuildError, build_authored_agent from .agent_type_check import EXIT_REFUSED from .backends import StageDone -from .contract import AgentContract from .failure_file import FAILURE_FILE from .framework_detection import detect_framework, named_framework_folder from .job import FailureDomain, HarnessJob -from .reading_entry import EXIT_DONE, EXIT_RETRYABLE, reading_failure, write_reading_failure +from .reading_entry import ( + EXIT_DONE, + EXIT_RETRYABLE, + MaskSecrets, + reading_failure, + write_reading_failure, +) +from .scenario_source import authored_agent_document if TYPE_CHECKING: from .agent_reading import _ReadingStage logger = logging.getLogger(__name__) +logger.addFilter(MaskSecrets()) AGENT_FILE = "agent.json" CONTRACT_FILE = "contract.json" @@ -189,25 +194,15 @@ def document(job: HarnessJob, source: Path, output: Path) -> int: contract_bytes = (output / CONTRACT_FILE).stat().st_size if contract_bytes > _CONTRACT_LIMIT_BYTES: raise _TooLarge({"contract_bytes": contract_bytes}) - contract = AgentContract.model_validate( - json.loads((output / CONTRACT_FILE).read_text(encoding="utf-8")) - ).model_dump(mode="json") - # The echo proves which folder was read, so it comes from the contract. - built_from = str((contract.get("runtime") or {}).get("source_subfolder") or "") - if built_from != (job.source.subfolder or ""): - raise AgentBuildError("subfolder_mismatch") framework = detect_framework( _framework_folder(job, source, output), repo_root=source, package_dir=None ) - inner = build_authored_agent( - contract, subfolder=built_from or None, framework=framework + # The build's own document, made by the build's own code. + made = authored_agent_document( + output / CONTRACT_FILE, subfolder=job.source.subfolder, framework=framework ) - body = json.dumps( - {"sdk_version": __version__, "agent": inner}, - ensure_ascii=False, - separators=(",", ":"), - ).encode("utf-8") - stored = json.dumps(inner, ensure_ascii=False).encode("utf-8") + body = json.dumps(made, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + stored = json.dumps(made["agent"], ensure_ascii=False).encode("utf-8") if len(body) > _DOCUMENT_LIMIT_BYTES or len(stored) > _DOCUMENT_LIMIT_BYTES: raise _TooLarge( { @@ -233,7 +228,7 @@ def document(job: HarnessJob, source: Path, output: Path) -> int: except Exception as exc: code = str(getattr(exc, "code", "") or "") logger.warning( - "agent document not built: class=%s code=%s", type(exc).__name__, code + "agent document not built: class=%s code=%s", type(exc).__name__, code, exc_info=True ) write_reading_failure( reading_failure( diff --git a/src/fi/alk/harness/scenario_source.py b/src/fi/alk/harness/scenario_source.py index 053e100d3..e9b7a19cd 100644 --- a/src/fi/alk/harness/scenario_source.py +++ b/src/fi/alk/harness/scenario_source.py @@ -645,6 +645,28 @@ def load_scenarios( return scenarios +def authored_agent_document( + contract_file: Path, *, subfolder: str | None, framework: Any +) -> dict[str, Any]: + """The agent document stored for a saved contract, ``{"sdk_version", "agent"}``. + + Both a build and a reading with no build make it here, so the two can't differ. + """ + contract = AgentContract.model_validate( + json.loads(contract_file.read_text(encoding="utf-8")) + ).model_dump(mode="json") + # The echo proves which folder was built, so it comes from the contract. + built_from = str((contract.get("runtime") or {}).get("source_subfolder") or "") + if built_from != (subfolder or ""): + raise AgentBuildError("agent_invalid", "subfolder_mismatch") + return { + "sdk_version": _sdk_version, + "agent": build_authored_agent( + contract, subfolder=built_from or None, framework=framework + ), + } + + class BundleScenarioSource: """The real `ScenarioSource`: reads and compiles the bundle's own scenario documents. `hosted_entrypoint.run_job` wires this in only when the injected source is still the default @@ -735,22 +757,11 @@ async def build( if source_kind in (SourceKind.GITHUB, SourceKind.ARCHIVE): framework = await self._detect_framework() try: - contract_file = bundle_dir / "contract.json" - contract = AgentContract.model_validate( - json.loads(contract_file.read_text(encoding="utf-8")) - ).model_dump(mode="json") - # The echo proves which folder was built, so it comes from the contract. - built_from = str( - (contract.get("runtime") or {}).get("source_subfolder") or "" + authored_agent = authored_agent_document( + bundle_dir / "contract.json", + subfolder=job.source.subfolder, + framework=framework, ) - if built_from != (job.source.subfolder or ""): - raise AgentBuildError("agent_invalid", "subfolder_mismatch") - authored_agent = { - "sdk_version": _sdk_version, - "agent": build_authored_agent( - contract, subfolder=built_from or None, framework=framework - ), - } except Exception as exc: # A field path can name the customer's own keys, so only the mismatch is named. mismatch = getattr(exc, "detail", "") == "subfolder_mismatch" diff --git a/tests/harness/test_agent_reading.py b/tests/harness/test_agent_reading.py index 192eaf997..48d654b68 100644 --- a/tests/harness/test_agent_reading.py +++ b/tests/harness/test_agent_reading.py @@ -8,11 +8,13 @@ import argparse import asyncio +import dataclasses import json import logging import os import shutil import threading +import types import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path @@ -25,6 +27,7 @@ authoring_entrypoint, reading_entry, reading_result, + scenario_source, session, spend, ) @@ -56,8 +59,22 @@ _LIMIT_ENV = "ALK_READING_MAX_DURATION_SECONDS" +class _LoudExit: + """``os`` for the helper under test: its hard limit ending this test process is a failure, + never the silent exit 0 its backstop gives a finished reading.""" + + def __getattr__(self, name): + return getattr(os, name) + + @staticmethod + def _exit(_code): + os.write(2, b"a reading's hard limit ended the test process\n") + os._exit(97) + + @pytest.fixture(autouse=True) def _fresh_state(monkeypatch, tmp_path): + monkeypatch.setattr(reading_entry, "os", _LoudExit()) monkeypatch.delenv(spend.JOURNAL_ALIAS, raising=False) monkeypatch.setenv("ALK_HARNESS_MODEL", "gemini-2.5-flash") # The machine's own secrets files must not decide what a test sees masked. @@ -89,6 +106,9 @@ async def start(self) -> None: raise FileNotFoundError("the Claude CLI was not found") async def stop(self) -> None: + if self.backend.close_hangs: + # A session stuck shutting down; only the loop's own teardown ends it. + await asyncio.Event().wait() output = self.backend.output self.backend.stopped.append( { @@ -128,6 +148,7 @@ def __init__(self, name: str, output: Path, turns: list) -> None: self.interrupt_says = "the interrupt control request timed out" self.start_raises = False self.saves_on_start: dict | None = None + self.close_hangs = False self.submissions = 0 def create(self, spec) -> _Live: @@ -479,6 +500,131 @@ def test_only_github_or_an_upload_is_read(tmp_path, model, kind): assert fake.created == [] +@pytest.mark.parametrize( + "agent", + [ + {"connector": "vapi", "mode": "provider_import", "config": {"assistant_id": "asst-1"}}, + {"connector": "vapi", "mode": "connect_only", "config": {"assistant_id": "asst-1"}}, + ], + ids=["provider_import", "connect_only"], +) +def test_a_provider_defined_agent_is_not_read_without_a_build(tmp_path, model, agent): + _repo(tmp_path) + fake = model() + output = tmp_path / "reading" + + assert _run(tmp_path, {**_job(), "agent": agent}) == 78 + + failure = _failure_in(output) + assert (failure["code"], failure["domain"], failure["retryable"]) == ( + READING_JOB_INVALID, + "platform_sync", + False, + ) + assert "repository only" in failure["message"] + assert fake.created == [] + + +def test_a_cancel_before_the_save_is_a_crash_with_its_file(tmp_path, model): + _repo(tmp_path) + + async def cancelled(live): + raise asyncio.CancelledError() + yield _done() + + model(cancelled) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_crashed", True) + assert failure["message"].startswith("CancelledError") + _assert_cost(output, 0.0, False) + + +def test_a_cancel_after_the_save_keeps_the_saved_contract(tmp_path, model): + _repo(tmp_path) + + async def saves_then_cancelled(live): + _save(live.backend.output, CHAT) + raise asyncio.CancelledError() + yield _done() + + model(saves_then_cancelled) + output = tmp_path / "reading" + + assert _run(tmp_path) == 0 + + assert _names(output) == {"contract.json", "agent.json", "cost.json"} + _assert_cost(output, 0.0, False) + + +def test_a_session_whose_close_hangs_still_ends_with_its_saved_contract( + tmp_path, model, monkeypatch, caplog +): + _repo(tmp_path) + _short_limit(monkeypatch) + + async def saves_then_goes_quiet(live): + _save(live.backend.output, CHAT) + await asyncio.Event().wait() + yield _done() + + fake = model(saves_then_goes_quiet) + fake.close_hangs = True + output = tmp_path / "reading" + job_path = tmp_path / "job.json" + job_path.write_text(json.dumps(_job()), encoding="utf-8") + # A short hard limit, so a reading stuck on the close ends this test (exit 97), not CI's clock. + mode = dataclasses.replace( + agent_reading._read_agent(interrupt_wait=0.3), limit_margin_seconds=5.0 + ) + + with caplog.at_level("WARNING", logger=agent_reading.__name__): + assert run_reading_mode(mode, job_path=job_path, source=tmp_path / "source", output=output) == 0 + + assert _names(output) == {"contract.json", "agent.json", "cost.json"} + # Decided while the close still hung: it never got as far as recording itself. + assert fake.stopped == [] + assert any("did not close" in record.getMessage() for record in caplog.records) + + +def test_at_the_hard_limit_a_saved_contract_ends_as_the_reading_ends_it(tmp_path): + source = _repo(tmp_path) + job = HarnessJob.model_validate(_job()) + output = tmp_path / "reading" + output.mkdir() + + assert agent_reading._at_the_limit(job, source, output) is None + _save(output, CHAT) + assert agent_reading._at_the_limit(job, source, output) == 0 + assert (output / "agent.json").exists() + assert agent_reading._at_the_limit(job, source, output) == 0 + assert agent_reading.READ_AGENT.at_limit is agent_reading._at_the_limit + + +def test_a_failed_document_is_logged_with_its_masked_trace(tmp_path, model, monkeypatch, caplog): + secret = "sk-document-0123456789" + secrets = tmp_path / "secrets.json" + secrets.write_text(json.dumps({"KEY": secret}), encoding="utf-8") + monkeypatch.setattr(reading_entry, "_SECRET_FILES", (secrets,)) + + def breaks(*_args, **_kwargs): + raise RuntimeError(f"the document builder broke on {secret}") + + monkeypatch.setattr(scenario_source, "build_authored_agent", breaks) + _repo(tmp_path) + model(_writes(CHAT)) + + with caplog.at_level("WARNING", logger=reading_result.__name__): + assert _run(tmp_path) == 78 + + [logged] = [record for record in caplog.records if record.name == reading_result.__name__] + assert "RuntimeError: the document builder broke on ***" in logged.exc_text + assert secret not in caplog.text + + def test_a_folder_mismatch_is_an_invalid_document(tmp_path, model): _repo(tmp_path) # Saved with no folder recorded, for a job that names one. @@ -489,7 +635,8 @@ def test_a_folder_mismatch_is_an_invalid_document(tmp_path, model): failure = _failure_in(output) assert (failure["code"], failure["retryable"]) == ("agent_document_invalid", False) - assert failure["details"]["code"] == "subfolder_mismatch" + # The build's own code for it, from the build's own document code. + assert failure["details"]["code"] == "agent_invalid" assert (output / "contract.json").exists() assert not (output / "agent.json").exists() _assert_cost(output, 0.11, True) @@ -533,7 +680,7 @@ def test_an_oversized_document_is_refused(tmp_path, model, monkeypatch, inner, p document = inner() if not passes and document["items"]: assert _body_size(document) <= MIB - monkeypatch.setattr(reading_result, "build_authored_agent", lambda *a, **k: document) + monkeypatch.setattr(scenario_source, "build_authored_agent", lambda *a, **k: document) model(_writes(CHAT, cost_usd=0.05)) output = tmp_path / "reading" @@ -1531,11 +1678,14 @@ def test_the_claude_backend_accepts_the_reading_spec(tmp_path, model, monkeypatc def test_the_soft_limit_counts_from_the_readings_start(tmp_path, model, monkeypatch): _repo(tmp_path) - _short_limit(monkeypatch, soft=3.0) + _short_limit(monkeypatch, soft=30.0) real = agent_reading.open_stage + now = [1000.0] + # The reading's own clock, so the time spent opening the stage is told, not slept. + monkeypatch.setattr(agent_reading, "time", types.SimpleNamespace(monotonic=lambda: now[0])) def slow(*args, **kwargs): - time.sleep(2.0) + now[0] += 29.9 return real(*args, **kwargs) monkeypatch.setattr(agent_reading, "open_stage", slow) @@ -1544,8 +1694,9 @@ def slow(*args, **kwargs): assert _run(tmp_path, wait=0.2) == 78 + # 0.1 s were left of the 30, not 30 again from the stage's opening. [interrupted] = fake.interrupts - assert 2.8 <= interrupted - started < 4.0 + assert interrupted - started < 10.0 # -- R37 to R41: the switch, the folders, the limit and the last write -------------------------- diff --git a/tests/harness/test_reading_entry.py b/tests/harness/test_reading_entry.py index 19bd049ef..b274d7363 100644 --- a/tests/harness/test_reading_entry.py +++ b/tests/harness/test_reading_entry.py @@ -41,8 +41,22 @@ } +class _LoudExit: + """``os`` for the helper under test: its hard limit ending this test process is a failure, + never the silent exit 0 its backstop gives a finished reading.""" + + def __getattr__(self, name): + return getattr(os, name) + + @staticmethod + def _exit(_code): + os.write(2, b"a reading's hard limit ended the test process\n") + os._exit(97) + + @pytest.fixture(autouse=True) def _fresh_state(monkeypatch, tmp_path): + monkeypatch.setattr(reading_entry, "os", _LoudExit()) monkeypatch.delenv(spend.JOURNAL_ALIAS, raising=False) # The machine's own secrets files must not decide what a test sees masked. monkeypatch.setattr(reading_entry, "_SECRET_FILES", (tmp_path / "no-secrets.json",)) @@ -165,6 +179,9 @@ async def work(job, source, output): reading_entry.write_reading_cost(output, complete=True) if config["work"] == "completes_then_raises": raise RuntimeError("the work broke after its last write") + if config["work"] == "saves_then_blocks": + # Saved, then stuck where a session's close can hang: past the limit, never returning. + (output / "contract.json").write_text(json.dumps(config["contract"])) if config["work"] == "spoils_the_output_then_blocks": # Every write at the limit now fails, and each failure would log. shutil.rmtree(output) @@ -178,7 +195,12 @@ async def work(job, source, output): return 0 -mode = ReadingMode(check_job=lambda job: None, work=work, **config["mode"]) +extra = {} +if config.get("at_limit") == "read_agent": + from fi.alk.harness import agent_reading + + extra["at_limit"] = agent_reading._at_the_limit +mode = ReadingMode(check_job=lambda job: None, work=work, **config["mode"], **extra) (root / "started").write_text(repr(time.time())) code = run_reading_mode( mode, job_path=root / "job.json", source=root / "source", output=root / "reading" @@ -190,7 +212,7 @@ async def work(job, source, output): _BLOCK_SECONDS = 20.0 -def _run_in_a_child(root: Path, mode_values: dict, work: str, *, linger: float = 0.0): +def _run_in_a_child(root: Path, mode_values: dict, work: str, *, linger: float = 0.0, **more): """Returns the finished child and the seconds from the helper's start to the child's end.""" root.mkdir(parents=True, exist_ok=True) (root / "job.json").write_text(json.dumps(_job()), encoding="utf-8") @@ -201,6 +223,7 @@ def _run_in_a_child(root: Path, mode_values: dict, work: str, *, linger: float = "work": work, "block_seconds": _BLOCK_SECONDS, "linger_seconds": linger, + **more, } env = { **os.environ, @@ -488,6 +511,28 @@ def _empty_spend() -> dict: return {"schema": "futureagi.harness-spend.v1", "total_usd": 0.0, "unpriced_turns": 0, "stages": []} +def test_a_contract_saved_before_the_hard_limit_still_ends_with_its_document(tmp_path): + (tmp_path / "source").mkdir(parents=True) + contract = { + "agent": "support", + "modality": "chat", + "real_use_cases": ["order"], + "tools": [{"name": "add", "args": ["item_id"]}], + } + mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} + + finished, elapsed = _run_in_a_child( + tmp_path, mode_values, "saves_then_blocks", contract=contract, at_limit="read_agent" + ) + + output = tmp_path / "reading" + assert finished.returncode == 0, finished.stderr + assert 1.0 <= elapsed <= 1.0 + 5 + assert not (output / "failure.json").exists() + assert json.loads((output / "agent.json").read_text())["agent"] + assert _cost_in(output)["complete"] is False + + def test_a_timeout_after_a_complete_cost_marks_it_incomplete(tmp_path): mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} From 4eb45c603701bbc7e50708dc28493cc0169b6c8c Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 13:08:17 +0530 Subject: [PATCH 12/23] test(harness): name the model-step test for when it really runs The model step also runs when the walk found agents only in test or example folders, so the test is no longer named "only when nothing is found". TH-8950 Signed-off-by: khushalsonawat --- tests/harness/test_agent_scan.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/harness/test_agent_scan.py b/tests/harness/test_agent_scan.py index 31a0ba2cb..7e29e9bc5 100644 --- a/tests/harness/test_agent_scan.py +++ b/tests/harness/test_agent_scan.py @@ -398,7 +398,9 @@ def test_an_uploads_ai_row_is_at_the_root(tmp_path, ai) -> None: assert document["commit_sha"] is None -def test_the_ai_runs_only_when_nothing_is_found(tmp_path, monkeypatch, ai) -> None: +def test_the_ai_runs_unless_an_agent_is_found_outside_side_folders(tmp_path, monkeypatch, ai) -> None: + """Agents only in test or example folders still ask the model, given the time; an agent + outside them doesn't. The weak-sign-beside-TS/JS case has its own tests below.""" side_only = _write(tmp_path / "side-only", SIDE_ONLY) monkeypatch.setattr(agent_scan, "CLOCK", 1000.0) From 45eb82ba5efdc9cfdd017fca681005913b19fae4 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 13:33:54 +0530 Subject: [PATCH 13/23] fix(harness): log the trace when the cost note after a turn fails The catch still keeps a paid answer from becoming a retry; it now logs the masked trace like the other absorbed faults, so a fault in the cost writer can be found. TH-8950 Signed-off-by: khushalsonawat --- src/fi/alk/harness/scan_ai.py | 10 ++++------ tests/harness/test_scan_ai.py | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/fi/alk/harness/scan_ai.py b/src/fi/alk/harness/scan_ai.py index 5e3874b71..0a49931aa 100644 --- a/src/fi/alk/harness/scan_ai.py +++ b/src/fi/alk/harness/scan_ai.py @@ -166,12 +166,10 @@ def _events(self, received: Any, turn: Turn) -> list[Any]: self.done.append(received) try: self._on_turn_end() - except Exception as exc: - # A failed cost note must not turn a paid answer into a retried one. - logger.warning( - "find_agents: the cost note after the turn failed (%s)", - type(exc).__name__, - ) + except Exception: + # A failed cost note must not turn a paid answer into a retried one. The + # trace is logged masked, so a fault in the cost writer can be found. + logger.warning("find_agents: the cost note after the turn failed", exc_info=True) # -- the run ----------------------------------------------------------------------------------- diff --git a/tests/harness/test_scan_ai.py b/tests/harness/test_scan_ai.py index 5421dd15b..59fc5851a 100644 --- a/tests/harness/test_scan_ai.py +++ b/tests/harness/test_scan_ai.py @@ -8,6 +8,7 @@ from __future__ import annotations import asyncio +import logging import os import time from dataclasses import dataclass, field @@ -1008,3 +1009,28 @@ def globbed(pattern: str, **extra: str) -> set[str]: assert len(scan_ai._pattern_forms("{a,b}" * 20)) <= scan_ai._MAX_FORMS found = _said(_call(tools["grep"], {"text": "POST", "glob": "**/*.{ts,tsx}"})) assert _listed(found) == {ROUTE} + + +def test_a_failed_cost_note_keeps_the_answer_and_logs_its_trace(tmp_path, monkeypatch, quick, caplog): + """A paid answer is never turned into a retry by the cost note, and the note's fault is + logged with its trace so it can be found.""" + backend = _Backend(KEPT, FINISHED) + monkeypatch.setattr(session, "resolve", lambda *_args, **_kwargs: backend) + + def broken_note() -> None: + raise OSError("cost file not writable") + + with caplog.at_level(logging.WARNING, logger=scan_ai.logger.name): + outcome = asyncio.run( + asyncio.wait_for( + scan_ai.run_ai_step( + _repo(tmp_path), hints="x", seconds=5.0, on_turn_end=broken_note + ), + TEST_BOUND, + ) + ) + + assert outcome.kind == "pairs" + assert outcome.pairs == (AGENT,) + assert "the cost note after the turn failed" in caplog.text + assert "Traceback" in caplog.text and "OSError" in caplog.text From 884043ca422eb260de6a267c2109f284afb841bf Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 13:45:24 +0530 Subject: [PATCH 14/23] fix(harness): keep describing a paid row past a bad file, and surface check faults From the review round on the fixes for the first review of this PR: - One unreadable or unparsable file no longer stops the description of a model-found agent. It marks the row short (partial) and the reading goes on, as the walk goes on past an unread file. - detect_framework takes raise_faults (off for builds). The scan's folder check turns it on, so a fault inside the framework check comes back as a trace and the row is unfinished, instead of a finished unknown. Both the child's and the parent's halves are tested. - A model tool's fault keeps the worker's exception as the cause, so its logged trace shows the real error. - Tests pin the run-time read of ALK_SCAN_MAX_SECONDS and the secret mask on every scan logger. The log comment says only the platform's secret values are masked. TH-8950 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_scan.py | 3 +- src/fi/alk/harness/framework_detection.py | 6 ++++ src/fi/alk/harness/scan_ai.py | 4 ++- src/fi/alk/harness/scan_folders.py | 1 + src/fi/alk/harness/scan_walk.py | 19 +++++++---- tests/harness/test_agent_scan.py | 21 ++++++++++++ tests/harness/test_scan_ai.py | 16 +++++++++ tests/harness/test_scan_folders.py | 40 +++++++++++++++++++++-- tests/harness/test_scan_walk.py | 26 +++++++++++++++ 9 files changed, 124 insertions(+), 12 deletions(-) diff --git a/src/fi/alk/harness/agent_scan.py b/src/fi/alk/harness/agent_scan.py index 4278a8480..c6856c2c7 100644 --- a/src/fi/alk/harness/agent_scan.py +++ b/src/fi/alk/harness/agent_scan.py @@ -389,7 +389,8 @@ def _keep_what_was_paid_for( job: HarnessJob, output: Path, scan: _Scan, crash: Exception ) -> int: """After the model was paid for, a crash still ends in a file, never in a paid retry.""" - # Logged masked, as the reading helper logs a crash. + # Logged with its trace, as the reading helper logs a crash. Only the platform's secret + # values are masked; a repo path in the message is not. logger.error("agent scan: failed after the model step", exc_info=crash) checked = {row.candidate.entry for row in scan.ai_rows} ai_rows = [ diff --git a/src/fi/alk/harness/framework_detection.py b/src/fi/alk/harness/framework_detection.py index 6b90fe5a6..6f531c425 100644 --- a/src/fi/alk/harness/framework_detection.py +++ b/src/fi/alk/harness/framework_detection.py @@ -949,7 +949,11 @@ def detect_framework( *, repo_root: Path | None = None, package_dir: Path | None = None, + raise_faults: bool = False, ) -> dict[str, Any]: + """The folder's framework word. A fault in the check is ``unknown`` for a build; the agent + scan passes ``raise_faults=True`` to see it, so it can log the trace and mark the row + unfinished instead of reporting a finished ``unknown``.""" try: if folder is None or not Path(folder).is_dir(): logger.info("framework detection: name=%s blockers=%s", "unknown", []) @@ -967,6 +971,8 @@ def detect_framework( ) return result except Exception as exc: + if raise_faults: + raise logger.warning("framework detection failed: class=%s", type(exc).__name__) return {"name": "unknown", "version": None} diff --git a/src/fi/alk/harness/scan_ai.py b/src/fi/alk/harness/scan_ai.py index 0a49931aa..b379fe808 100644 --- a/src/fi/alk/harness/scan_ai.py +++ b/src/fi/alk/harness/scan_ai.py @@ -391,8 +391,10 @@ def run() -> None: result = work(*args) except _Refused as refused: error = refused - except BaseException: + except BaseException as exc: error = _ToolFailed() + # Kept as the cause, so the logged trace shows the worker's own fault. + error.__cause__ = exc try: loop.call_soon_threadsafe(settle, result, error) except RuntimeError: diff --git a/src/fi/alk/harness/scan_folders.py b/src/fi/alk/harness/scan_folders.py index 148244fa5..3fb460e7f 100644 --- a/src/fi/alk/harness/scan_folders.py +++ b/src/fi/alk/harness/scan_folders.py @@ -60,6 +60,7 @@ def _answer(source: Path, subfolder: str) -> dict[str, Any]: agent_folder(source, subfolder) if subfolder else source, repo_root=source, package_dir=None, + raise_faults=True, ) except SubfolderNotFound: # A folder the build would refuse has no word; it is not a reason to end the child. diff --git a/src/fi/alk/harness/scan_walk.py b/src/fi/alk/harness/scan_walk.py index 90a656357..be62c8163 100644 --- a/src/fi/alk/harness/scan_walk.py +++ b/src/fi/alk/harness/scan_walk.py @@ -1237,7 +1237,11 @@ def __init__( self.prompt_files = 0 self.files_read = 0 self.bytes_read = 0 + # A cap or the clock: the reading stops here. self.partial = False + # One file that couldn't be read or parsed: the counts are short, but the reading + # goes on, as the walk goes on past an unread file. + self.short = False self.shadowed = _ShadowedNames(top, self.projects) def run(self) -> None: @@ -1257,12 +1261,12 @@ def run(self) -> None: self.templates |= _template_names(data) except _FILE_ERRORS: # Its names are missing, so the row's settings are short. - self.partial = True + self.short = True except MemoryError: raise except Exception: logger.warning("agent description: a template's reader failed", exc_info=True) - self.partial = True + self.short = True start = os.path.join(self.top, self.subfolder) if self.subfolder else self.top for directory, folder, name in _source_files(self.top, start): if self._out_of_time(): @@ -1296,12 +1300,12 @@ def _take(self, directory: str, folder: str, name: str) -> None: self.script_env |= _script_names(data) except _FILE_ERRORS: # Its facts are missing, so the row's counts are short. - self.partial = True + self.short = True except MemoryError: raise except Exception: logger.warning("agent description: a file's reader failed", exc_info=True) - self.partial = True + self.short = True def _read(self, path: str) -> bytes | None: """The file's bytes, or None for a file that can't or mustn't be read; sets @@ -1317,7 +1321,7 @@ def _read(self, path: str) -> bytes | None: except (OSError, ValueError): # Unreadable, a link, or a name the system can't open: not read, so the counts # are short. - self.partial = True + self.short = True return None self.files_read += 1 self.bytes_read += len(data) @@ -1364,7 +1368,8 @@ def describe_entry( code = {name for found in reading.facts.values() for name in found.env} code |= reading.script_env project_file = projects.file_in(subfolder) - if not reading.partial: + partial = reading.partial or reading.short + if not partial: logger.info("agent description: files_read=%d", reading.files_read) else: logger.info("agent description: files_read=%d partial=yes", reading.files_read) @@ -1377,7 +1382,7 @@ def describe_entry( build_may_stop=build_hazard( source, subfolder, deadline=deadline, clock=clock, hazards=hazards ), - partial=reading.partial, + partial=partial, env_names=(*sorted(code), *sorted(reading.templates - code)), tools=sum(found.tools for found in reading.facts.values()), prompt_files=reading.prompt_files, diff --git a/tests/harness/test_agent_scan.py b/tests/harness/test_agent_scan.py index 7e29e9bc5..130e40865 100644 --- a/tests/harness/test_agent_scan.py +++ b/tests/harness/test_agent_scan.py @@ -1187,3 +1187,24 @@ def broken(*args, **kwargs): assert (failure["code"], failure["retryable"]) == ("scan_crashed", True) assert not (output / "agents.json").exists() assert len(ai.calls) == 0 + + +def test_the_scan_reads_its_clock_variable_when_it_starts(tmp_path, monkeypatch, ai) -> None: + """The helper's hard stop and the scan's own clock read ALK_SCAN_MAX_SECONDS at the same + moment, so a shorter clock set at run time ends the scan in its own file.""" + source = _write(tmp_path / "source", {"a/graph.py": GRAPH}) + monkeypatch.setattr(agent_scan, "CLOCK", 1200.0) + monkeypatch.setenv("ALK_SCAN_MAX_SECONDS", "0.000001") + + assert _scan(_job(), source, tmp_path / "out") == 0 + + assert _agents(tmp_path / "out")["error"]["code"] == "scan_timed_out" + assert ai.calls == [] + + +def test_every_scan_logger_masks_secrets() -> None: + from fi.alk.harness import scan_ai, scan_folders + from fi.alk.harness.reading_entry import MaskSecrets + + for module in (agent_scan, scan_walk, scan_folders, scan_ai): + assert any(isinstance(f, MaskSecrets) for f in module.logger.filters), module.__name__ diff --git a/tests/harness/test_scan_ai.py b/tests/harness/test_scan_ai.py index 59fc5851a..475aa1e9f 100644 --- a/tests/harness/test_scan_ai.py +++ b/tests/harness/test_scan_ai.py @@ -1034,3 +1034,19 @@ def broken_note() -> None: assert outcome.pairs == (AGENT,) assert "the cost note after the turn failed" in caplog.text assert "Traceback" in caplog.text and "OSError" in caplog.text + + +def test_a_tool_fault_logs_the_workers_own_error(tmp_path, monkeypatch, caplog): + """The model is told only that the call failed; the log carries the worker's real fault.""" + + def broken(root, arguments): + raise KeyError("a fault in the glob worker") + + monkeypatch.setattr(scan_ai, "_glob_files", broken) + tools = _tools(_repo(tmp_path)) + with caplog.at_level(logging.WARNING, logger=scan_ai.logger.name): + said = _said(_call(tools["glob"], {"pattern": "*"})) + + assert said == scan_ai._FAILED + assert "a tool call failed" in caplog.text + assert "KeyError" in caplog.text and "a fault in the glob worker" in caplog.text diff --git a/tests/harness/test_scan_folders.py b/tests/harness/test_scan_folders.py index 0baa33a23..079dd6666 100644 --- a/tests/harness/test_scan_folders.py +++ b/tests/harness/test_scan_folders.py @@ -223,15 +223,49 @@ def refuse(pid: int, sig: int) -> None: def test_a_fault_in_the_check_comes_back_as_a_trace(tmp_path, monkeypatch) -> None: - """Only a folder the build refuses is a plain ``unknown``; any other fault is sent back - with its trace, so the parent logs it and marks the row unfinished.""" + """Only a folder the build refuses is a plain ``unknown``; a fault inside the framework + check itself (here in ``folder_findings``, below detect_framework's own catch) is sent + back with its trace, so the parent logs it and marks the row unfinished.""" + import fi.alk.harness.framework_detection as fd def broken(*args, **kwargs): raise KeyError("a fault in the check") - monkeypatch.setattr(scan_folders, "detect_framework", broken) + monkeypatch.setattr(fd, "folder_findings", broken) + (tmp_path / "agent.py").write_text(CREW, encoding="utf-8") + assert fd.detect_framework(tmp_path) == {"name": "unknown", "version": None} answer = scan_folders._answer(tmp_path, "") assert set(answer) == {scan_folders._FAILED} assert "KeyError" in answer[scan_folders._FAILED] assert scan_folders._answer(tmp_path, "no/such/folder") == {"name": "unknown", "version": None} + + +def test_a_failed_answer_marks_the_row_unfinished_and_logs_the_trace(tmp_path, caplog) -> None: + """The parent's half: a child answer carrying a trace becomes an unfinished ``unknown``, + kept for the folder, with the trace in the log.""" + + class _Conn: + def send(self, message: object) -> None: + pass + + def poll(self, timeout: float) -> bool: + return True + + def recv(self) -> object: + return {scan_folders._FAILED: "Traceback (most recent call last):\nKeyError: 'x'"} + + checker = FolderChecker() + checker._process = object() # a started child, as far as check() can tell + checker._conn = _Conn() + try: + with caplog.at_level("WARNING", logger=scan_folders.logger.name): + word = checker.check(tmp_path, "", deadline=time.monotonic() + 30) + finally: + checker._process = None + checker._conn = None + + assert word.finished is False + assert word.framework["name"] == "unknown" + assert "the check failed in the child" in caplog.text and "KeyError" in caplog.text + assert checker.check(tmp_path, "", deadline=time.monotonic() + 30).finished is False diff --git a/tests/harness/test_scan_walk.py b/tests/harness/test_scan_walk.py index cd92ea374..bd8210513 100644 --- a/tests/harness/test_scan_walk.py +++ b/tests/harness/test_scan_walk.py @@ -1225,3 +1225,29 @@ def test_side_folders_cannot_use_up_the_file_cap_before_the_agent( assert "zz_agent/agent.py" in _entries(result) assert result.stopped == "limit" + + +@pytest.mark.parametrize("shape", ["linked_template", "unterminated_py", "long_line_py"]) +def test_one_unreadable_file_does_not_stop_an_ai_rows_description(tmp_path: Path, shape: str) -> None: + """The row was paid for: a bad file marks it partial, and the reading goes on.""" + _write( + tmp_path, + { + "app/bot.py": GRAPH + "import os\nos.getenv('OPENAI_API_KEY')\n", + "app/z_tool.py": "import os\nos.getenv('BOT_MODEL')\n", + }, + ) + if shape == "linked_template": + (tmp_path / "outside.env").write_text("SEARCH_KEY=x\n", encoding="utf-8") + (tmp_path / "app/.env.example").symlink_to(tmp_path / "outside.env") + elif shape == "unterminated_py": + (tmp_path / "app/a_gen.py").write_text("x = '''never closed\n", encoding="utf-8") + else: + (tmp_path / "app/a_gen.py").write_text("x = 1 # " + "y" * 70_000 + "\n", encoding="utf-8") + + candidate = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) + + assert candidate is not None + assert candidate.partial is True + assert {"OPENAI_API_KEY", "BOT_MODEL"} <= set(candidate.env_names) + assert "langgraph" in candidate.builds From 6aeb4665837df281c88193f39652a05746daa29a Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 13:51:40 +0530 Subject: [PATCH 15/23] fix(harness): let the walk reach a big repo's agent, and read templates with own code From the review round on the fixes for the first review of this PR: - The file cap is 200,000, up from 20,000. Measured: 100,000 tiny source files walk in 13 s at 266 MB, so the byte cap and the clock are what bound the walk. A repo whose own code passes 20,000 files no longer stops before its agent is read. - A folder's settings template (.env.example and its kin) is read in its folder's pass, not with the side files, so a big tests folder can't keep it from being read. - A reader fault of an unforeseen kind logs its trace once; the rest are counted (reader_faults in the walk's summary line). - ALK_SCAN_AI_MAX_SECONDS is read when the scan starts, as the clock is. - Tests: every file is read in exactly one pass; tests no longer depend on an exported ALK_SCAN_MAX_SECONDS; the process-wide idle-retry setting is set through monkeypatch. TH-8950 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_scan.py | 6 ++- src/fi/alk/harness/scan_walk.py | 28 ++++++++++--- tests/harness/test_agent_scan.py | 3 ++ tests/harness/test_scan_ai.py | 4 +- tests/harness/test_scan_walk.py | 67 +++++++++++++++++++++++++++++++- 5 files changed, 97 insertions(+), 11 deletions(-) diff --git a/src/fi/alk/harness/agent_scan.py b/src/fi/alk/harness/agent_scan.py index c6856c2c7..e5cda067f 100644 --- a/src/fi/alk/harness/agent_scan.py +++ b/src/fi/alk/harness/agent_scan.py @@ -61,7 +61,9 @@ # The scan's clock when CLOCK_ENV doesn't name one. The helper reads the same variable for its # hard stop, both when the reading starts. CLOCK = 1200.0 -AI_SECONDS = limit_seconds("ALK_SCAN_AI_MAX_SECONDS", 600.0) +AI_ENV = "ALK_SCAN_AI_MAX_SECONDS" +# The model step's limit when AI_ENV doesn't name one; read when the scan starts, as the clock is. +AI_SECONDS = 600.0 # The helper's hard stop comes this long after the scan's own clock, so a scan that stops itself # always writes its own file first. HARD_LIMIT_MARGIN = 600.0 @@ -492,7 +494,7 @@ async def _scan(job: HarnessJob, source: Path, output: Path, scan: _Scan) -> int outcome = await run_ai_step( source, hints=_hints(walk), - seconds=min(AI_SECONDS, deadline - clock() - AI_RESERVE), + seconds=min(limit_seconds(AI_ENV, AI_SECONDS), deadline - clock() - AI_RESERVE), on_turn_end=lambda: write_reading_cost(output, complete=False), ) if not outcome.complete: diff --git a/src/fi/alk/harness/scan_walk.py b/src/fi/alk/harness/scan_walk.py index be62c8163..1ecc9af9d 100644 --- a/src/fi/alk/harness/scan_walk.py +++ b/src/fi/alk/harness/scan_walk.py @@ -56,7 +56,9 @@ # A hundredth of a folder check's allowance: the crafted files that use a whole one # take seconds, so one file now costs about a second at most. _FILE_WORK = 1_000_000_000 -_MAX_FILES = 20_000 +# Measured 10 Nov: 100,000 tiny source files walk in 13 s at 266 MB, so this sits above any +# real repo; the byte cap and the clock are what bound the walk. +_MAX_FILES = 200_000 _MAX_BYTES = 268_435_456 _NAMES_PER_FILE = 200 _IMPORT_DEPTH = 10 @@ -733,10 +735,20 @@ def _source_files( directories[:] = [name for name in directories if not _is_side_part(name)] folder = "/".join(relative) for name in files: - if side is None or _is_side_path(_join(folder, name)) is side: + if side is None or _read_with_side_files(_join(folder, name)) is side: yield directory, folder, name +def _read_with_side_files(path: str) -> bool: + """Which pass reads ``path``. A settings template (``.env.example``) starts with a dot, which + makes it a side file by name; it belongs to its folder's pass, so a big tests folder can't + keep it from being read.""" + parts = path.split("/") + if parts[-1] in _TEMPLATES: + return any(_is_side_part(part) for part in parts[:-1]) + return _is_side_path(path) + + class _ShadowedNames: """Each project's own module names, read once per project, so an import of the customer's own ``agents`` package isn't read as a framework.""" @@ -776,6 +788,8 @@ def __init__( self.templates: dict[str, set[str]] = {} self.prompt_paths: list[str] = [] self.unread = 0 + # Unread files whose reader raised a kind not foreseen (a kit fault, not the file's). + self.faults = 0 self.files_read = 0 self.bytes_read = 0 self.stopped: Literal["", "limit", "clock", "memory"] = "" @@ -831,8 +845,11 @@ def _take(self, directory: str, folder: str, name: str) -> None: raise except Exception: # A reader fault of a kind not foreseen is still this file's problem, never the - # walk's: the file is unread, and the trace is logged masked. - logger.warning("agent walk: a file's reader failed", exc_info=True) + # walk's: the file is unread. The first trace is logged masked; the rest are + # counted, so a fault that hits every file can't flood the log. + if not self.faults: + logger.warning("agent walk: a file's reader failed", exc_info=True) + self.faults += 1 self.unread += 1 def _read(self, path: str) -> bytes | None: @@ -1145,9 +1162,10 @@ def walk_repo( # path, never to "the repository is larger than the scan reads". result = walk.result(upload=upload) logger.info( - "agent walk: files_read=%d unread=%d agents=%d left_out=%d stopped=%s", + "agent walk: files_read=%d unread=%d reader_faults=%d agents=%d left_out=%d stopped=%s", walk.files_read, result.unread_files, + walk.faults, len(result.agents), result.paths_left_out, result.stopped or "no", diff --git a/tests/harness/test_agent_scan.py b/tests/harness/test_agent_scan.py index 130e40865..9965f1d0b 100644 --- a/tests/harness/test_agent_scan.py +++ b/tests/harness/test_agent_scan.py @@ -103,6 +103,9 @@ def _fresh_state(monkeypatch, tmp_path): monkeypatch.setattr(reading_entry, "_SECRET_FILES", (tmp_path / "no-secrets.json",)) monkeypatch.setattr(agent_scan, "CLOCK", 1200.0) monkeypatch.setattr(agent_scan, "AI_SECONDS", 600.0) + # The scan reads these when it starts, so the machine's own settings mustn't decide a test. + monkeypatch.delenv(agent_scan.CLOCK_ENV, raising=False) + monkeypatch.delenv(agent_scan.AI_ENV, raising=False) spend._stages.clear() yield spend._stages.clear() diff --git a/tests/harness/test_scan_ai.py b/tests/harness/test_scan_ai.py index 475aa1e9f..e53825f6a 100644 --- a/tests/harness/test_scan_ai.py +++ b/tests/harness/test_scan_ai.py @@ -325,7 +325,7 @@ def test_each_ai_outcome_gives_its_result(tmp_path, monkeypatch, quick, name): case = _OUTCOMES[name] if case.idle is not None: monkeypatch.setattr(scan_ai, "IDLE_SECONDS", case.idle) - session.STAGE_IDLE_RETRIES = 1 + monkeypatch.setattr(session, "STAGE_IDLE_RETRIES", 1) backend = _Backend(*case.steps, interrupt=case.interrupt) ends: list = [] @@ -410,7 +410,7 @@ async def generate_content_async(self, llm_request, stream=False): def test_an_idle_ai_stream_is_not_replayed(tmp_path, monkeypatch, quick): - session.STAGE_IDLE_RETRIES = 1 + monkeypatch.setattr(session, "STAGE_IDLE_RETRIES", 1) monkeypatch.setattr(scan_ai, "IDLE_SECONDS", 0.2) backend = _Backend(("quiet",)) diff --git a/tests/harness/test_scan_walk.py b/tests/harness/test_scan_walk.py index bd8210513..9d34cbd7a 100644 --- a/tests/harness/test_scan_walk.py +++ b/tests/harness/test_scan_walk.py @@ -342,7 +342,7 @@ def test_a_broken_langgraph_json_is_unread_not_a_crash( def test_the_walk_limit_and_the_clock_stop_it( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - assert (scan_walk._MAX_FILES, scan_walk._MAX_BYTES) == (20_000, 268_435_456) + assert (scan_walk._MAX_FILES, scan_walk._MAX_BYTES) == (200_000, 268_435_456) _write(tmp_path, {"a.py": GRAPH, "b.py": GRAPH, "c.py": GRAPH}) calls: list[int] = [] @@ -361,7 +361,7 @@ def clock() -> float: assert by_count.stopped == "limit" assert _entries(by_count) == ["a.py", "b.py"] - monkeypatch.setattr(scan_walk, "_MAX_FILES", 20_000) + monkeypatch.setattr(scan_walk, "_MAX_FILES", 200_000) monkeypatch.setattr(scan_walk, "_MAX_BYTES", len(GRAPH)) by_bytes = _walk(tmp_path) assert by_bytes.stopped == "limit" @@ -1251,3 +1251,66 @@ def test_one_unreadable_file_does_not_stop_an_ai_rows_description(tmp_path: Path assert candidate.partial is True assert {"OPENAI_API_KEY", "BOT_MODEL"} <= set(candidate.env_names) assert "langgraph" in candidate.builds + + +def test_every_file_is_read_in_exactly_one_pass(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The two passes split the files: together they give each file once.""" + _write( + tmp_path, + { + "agent.py": GRAPH, + "app/.env.example": "A=1\n", + "app/test_helpers.py": "x = 1\n", + "app/conftest.py": "x = 1\n", + "tests/test_a.py": "x = 1\n", + "tests/.env.example": "B=1\n", + "examples/demo/agent.py": GRAPH, + "pkg/sub/mod.py": "x = 1\n", + "pkg/sub/tests/test_b.py": "x = 1\n", + }, + ) + seen: list[str] = [] + take = scan_walk._Walk._take + + def counting(self, directory, folder, name): + seen.append(scan_walk._join(folder, name)) + return take(self, directory, folder, name) + + monkeypatch.setattr(scan_walk._Walk, "_take", counting) + _walk(tmp_path) + + every = sorted(p.relative_to(tmp_path).as_posix() for p in tmp_path.rglob("*") if p.is_file()) + assert sorted(seen) == every + assert len(seen) == len(set(seen)) + # The customer's own code and its settings template come before any side file. + own = seen[: seen.index("examples/demo/agent.py")] + assert "app/.env.example" in own and "agent.py" in own + + +def test_a_template_beside_the_agent_is_read_before_a_big_tests_folder( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(scan_walk, "_MAX_FILES", 10) + _write(tmp_path, {f"tests/test_{index:02}.py": "x = 1\n" for index in range(25)}) + _write(tmp_path, {"zz_agent/agent.py": GRAPH, "zz_agent/.env.example": "OPENAI_API_KEY=\n"}) + + result = _walk(tmp_path) + + assert "OPENAI_API_KEY" in _agent(result, "zz_agent/agent.py").env_names + + +def test_a_reader_fault_on_every_file_logs_one_trace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + _write(tmp_path, {f"m{index}.py": "x = 1\n" for index in range(5)}) + + def broken(data: bytes, shadowed: object) -> object: + raise LookupError("a fault in the kit's reader") + + monkeypatch.setattr(scan_walk, "_python_facts", broken) + with caplog.at_level("WARNING", logger=scan_walk.logger.name): + result = _walk(tmp_path) + + assert result.unread_files == 5 + assert caplog.text.count("a file's reader failed") == 1 + assert "LookupError" in caplog.text From ef177b36ccf04c0865f319a4e39f505ad60cdde1 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 14:07:33 +0530 Subject: [PATCH 16/23] fix(harness): end a hung close before the hard limit instead of a second document build - The hook that built agent.json from the timer's thread is gone: two document builds could starve each other past the limit. The close after a stop is bounded at 20 s, so the reading decides from its files well before the hard limit. - The contract is read once and checked from the same bytes; a mismatch names subfolder_mismatch; a validation error logs no trace; a cancel during the stop's interrupt stays a timeout. TH-8951 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_reading.py | 28 ++-- src/fi/alk/harness/reading_entry.py | 56 ++------ src/fi/alk/harness/reading_result.py | 23 +++- src/fi/alk/harness/scenario_source.py | 11 +- tests/harness/test_agent_reading.py | 177 ++++++++++++++++++++++++-- tests/harness/test_reading_entry.py | 35 +---- 6 files changed, 214 insertions(+), 116 deletions(-) diff --git a/src/fi/alk/harness/agent_reading.py b/src/fi/alk/harness/agent_reading.py index eb5f6b29b..64c0607ad 100644 --- a/src/fi/alk/harness/agent_reading.py +++ b/src/fi/alk/harness/agent_reading.py @@ -24,7 +24,6 @@ from .failure_file import FAILURE_FILE from .job import FailureDomain, HarnessFailure, HarnessJob, ProviderExecutionMode, SourceKind from .reading_entry import ( - EXIT_DONE, EXIT_RETRYABLE, READING_JOB_INVALID, MaskSecrets, @@ -35,7 +34,6 @@ write_reading_failure, ) from .reading_result import ( - AGENT_FILE, CONTRACT_FILE, classify, document, @@ -62,6 +60,10 @@ _HARD_LIMIT_MARGIN_SECONDS = 180.0 _INTERRUPT_SECONDS = 10.0 _INTERRUPT_WAIT_SECONDS = 60.0 +# The SDK's own close gives up after about 20 s. With it, the decision comes by soft + 90 s, which +# leaves the saved contract's document (its framework scan, about 70 s at the scanner's cap) room +# to finish inside the hard limit's margin. +_CLOSE_SECONDS = 20.0 _COUNTS = ("tokens_in", "tokens_out", "tokens_cached") @@ -183,16 +185,18 @@ async def _stop( stopping.set() try: await asyncio.wait_for(stage.interrupt_response(), _INTERRUPT_SECONDS) - except Exception: + except (Exception, asyncio.CancelledError): # A stop that can't be delivered is still a timeout, not a crash. logger.warning("the stop at the time limit was not delivered", exc_info=True) await asyncio.wait({task}, timeout=wait) - # Closed before the files are read, so no late tool call can save after the decision. The - # close is bounded too: a session that hangs closing must not keep the reading from deciding. + # Closed before the files are read, so a late tool call is unlikely to save after the + # decision. The close is bounded, though: a session that hangs closing must not keep the + # reading from deciding, and one that outlives the bound could still act afterwards. task.cancel() - await asyncio.wait({task}, timeout=wait) + closing = min(wait, _CLOSE_SECONDS) + await asyncio.wait({task}, timeout=closing) if not task.done(): - logger.warning("the stopped reading's session did not close within %g s", wait) + logger.warning("the stopped reading's session did not close within %g s", closing) elif not task.cancelled() and task.exception() is not None: logger.warning("the stopped reading ended with an error", exc_info=task.exception()) @@ -323,15 +327,6 @@ async def _read(job: HarnessJob, source: Path, output: Path, *, interrupt_wait: return code -def _at_the_limit(job: HarnessJob, source: Path, output: Path) -> int | None: - """At the helper's hard limit: a saved contract still ends as the reading ends one.""" - if saved_file(output) != CONTRACT_FILE: - return None - if (output / AGENT_FILE).is_file(): - return EXIT_DONE - return document(job, Path(source), output) - - def _read_agent(*, interrupt_wait: float = _INTERRUPT_WAIT_SECONDS) -> ReadingMode: return ReadingMode( name="read_agent", @@ -345,7 +340,6 @@ def _read_agent(*, interrupt_wait: float = _INTERRUPT_WAIT_SECONDS) -> ReadingMo crashed_code=AGENT_READING_CRASHED, limit_env=_LIMIT_ENV, limit_margin_seconds=_HARD_LIMIT_MARGIN_SECONDS, - at_limit=_at_the_limit, ) diff --git a/src/fi/alk/harness/reading_entry.py b/src/fi/alk/harness/reading_entry.py index 9a0dd66c4..810ae820f 100644 --- a/src/fi/alk/harness/reading_entry.py +++ b/src/fi/alk/harness/reading_entry.py @@ -38,8 +38,6 @@ EXIT_DONE = 0 EXIT_RETRYABLE = 1 -# How long a mode's own end at the hard limit may take before the timeout is written anyway. -_SETTLE_SECONDS = 60.0 # Neither file is deleted by the hosted authoring entrypoint, so a read here takes nothing away # from the code that loads them. @@ -66,9 +64,6 @@ class ReadingMode: # ``time_limit_seconds`` as its default; the helper's own limit adds the margin to it. limit_env: str | None = None limit_margin_seconds: float = 0.0 - # Called at the hard limit, before any timeout is written: an exit code means the mode has - # already ended what it saved (its own files written), and no timeout is written over it. - at_limit: Callable[[HarnessJob, Path, Path], int | None] | None = None def limit_seconds(env: str, default: float) -> float: @@ -243,12 +238,8 @@ class _HardLimit: """Ends the process at the limit. Before the work returns, a timeout is written first; after it, the work's files stand and the process ends with the work's own code.""" - def __init__( - self, mode: ReadingMode, job: HarnessJob, source: Path, output: Path, limit: float - ) -> None: + def __init__(self, mode: ReadingMode, output: Path, limit: float) -> None: self._mode = mode - self._job = job - self._source = source self._output = output self._limit = limit self._returned: int | None = None @@ -273,44 +264,21 @@ def _end_the_process(self) -> None: # Nothing logs on the way out: a blocked stderr must not hold off the exit. logging.disable(logging.CRITICAL) try: - settled = self._settled() - if settled is not None: - # The mode ended what it had saved; a timeout would throw away a paid result. - exit_code = settled - else: - write_reading_failure( - reading_failure( - FailureDomain.INFRASTRUCTURE, - mode.timed_out_code, - f"The {mode.name} reading ran past its {limit:g} s limit " - "and was stopped.", - retryable=mode.timed_out_retryable, - ), - output, - ) + write_reading_failure( + reading_failure( + FailureDomain.INFRASTRUCTURE, + mode.timed_out_code, + f"The {mode.name} reading ran past its {limit:g} s limit " + "and was stopped.", + retryable=mode.timed_out_retryable, + ), + output, + ) write_reading_cost(output, complete=False) finally: # Work running in a thread can't be cancelled, and a normal exit would wait for it. os._exit(exit_code) - def _settled(self) -> int | None: - at_limit = self._mode.at_limit - if at_limit is None: - return None - settled: list[int | None] = [] - - def settle() -> None: - try: - settled.append(at_limit(self._job, self._source, self._output)) - except Exception: - pass - - # Its own thread, with a bound, so the mode's last step can't keep the process alive. - worker = threading.Thread(target=settle, daemon=True) - worker.start() - worker.join(_SETTLE_SECONDS) - return settled[0] if settled else None - async def _work_then_stop_the_clock( mode: ReadingMode, job: HarnessJob, source: Path, output: Path, limit: _HardLimit | None @@ -392,7 +360,7 @@ def run_reading_mode(mode: ReadingMode, *, job_path: Path, source: Path, output: # Read now, not when the mode was made, so it is the same limit the work reads as it starts. seconds = _hard_limit(mode) - limit = None if seconds is None else _HardLimit(mode, job, source, output, seconds) + limit = None if seconds is None else _HardLimit(mode, output, seconds) crash: BaseException | None = None try: logger.info("reading started at %s", datetime.now(timezone.utc).isoformat()) diff --git a/src/fi/alk/harness/reading_result.py b/src/fi/alk/harness/reading_result.py index 9f92fc9bd..496f4e29a 100644 --- a/src/fi/alk/harness/reading_result.py +++ b/src/fi/alk/harness/reading_result.py @@ -14,6 +14,8 @@ from pathlib import Path from typing import TYPE_CHECKING, Any +from pydantic import ValidationError + from .agent_type_check import EXIT_REFUSED from .backends import StageDone from .failure_file import FAILURE_FILE @@ -191,15 +193,20 @@ def document(job: HarnessJob, source: Path, output: Path) -> int: """The agent's description from the saved contract, as a build makes it.""" # One catch over every step: a saved contract is paid for, so nothing after it is retryable. try: - contract_bytes = (output / CONTRACT_FILE).stat().st_size + # Read once, here, so what is checked is what is used: the session may still save while + # the framework scan below runs. + with (output / CONTRACT_FILE).open("rb") as handle: + raw = handle.read(_CONTRACT_LIMIT_BYTES + 1) + contract_bytes = max(os.fstat(handle.fileno()).st_size, len(raw)) if contract_bytes > _CONTRACT_LIMIT_BYTES: raise _TooLarge({"contract_bytes": contract_bytes}) + contract_text = raw.decode("utf-8") framework = detect_framework( _framework_folder(job, source, output), repo_root=source, package_dir=None ) # The build's own document, made by the build's own code. made = authored_agent_document( - output / CONTRACT_FILE, subfolder=job.source.subfolder, framework=framework + contract_text, subfolder=job.source.subfolder, framework=framework ) body = json.dumps(made, ensure_ascii=False, separators=(",", ":")).encode("utf-8") stored = json.dumps(made["agent"], ensure_ascii=False).encode("utf-8") @@ -228,8 +235,16 @@ def document(job: HarnessJob, source: Path, output: Path) -> int: except Exception as exc: code = str(getattr(exc, "code", "") or "") logger.warning( - "agent document not built: class=%s code=%s", type(exc).__name__, code, exc_info=True + "agent document not built: class=%s code=%s", + type(exc).__name__, + code, + # pydantic's trace repeats values from the contract, which the mask can't know. + exc_info=not isinstance(exc, ValidationError), ) + details = {"class": type(exc).__name__, "code": code} + if getattr(exc, "detail", "") == "subfolder_mismatch": + # Named as the build names it: the one detail that is ours, not the contract's. + details["detail"] = "subfolder_mismatch" write_reading_failure( reading_failure( FailureDomain.INFRASTRUCTURE, @@ -237,7 +252,7 @@ def document(job: HarnessJob, source: Path, output: Path) -> int: "The agent's description could not be built from its saved contract " f"({type(exc).__name__}{': ' + code if code else ''}).", retryable=False, - details={"class": type(exc).__name__, "code": code}, + details=details, ), output, ) diff --git a/src/fi/alk/harness/scenario_source.py b/src/fi/alk/harness/scenario_source.py index e9b7a19cd..4d01f76da 100644 --- a/src/fi/alk/harness/scenario_source.py +++ b/src/fi/alk/harness/scenario_source.py @@ -646,15 +646,14 @@ def load_scenarios( def authored_agent_document( - contract_file: Path, *, subfolder: str | None, framework: Any + contract_text: str, *, subfolder: str | None, framework: Any ) -> dict[str, Any]: """The agent document stored for a saved contract, ``{"sdk_version", "agent"}``. - Both a build and a reading with no build make it here, so the two can't differ. + Both a build and a reading with no build make it here, so the two can't differ. The caller + reads the contract, so what it checked is what is used. """ - contract = AgentContract.model_validate( - json.loads(contract_file.read_text(encoding="utf-8")) - ).model_dump(mode="json") + contract = AgentContract.model_validate(json.loads(contract_text)).model_dump(mode="json") # The echo proves which folder was built, so it comes from the contract. built_from = str((contract.get("runtime") or {}).get("source_subfolder") or "") if built_from != (subfolder or ""): @@ -758,7 +757,7 @@ async def build( framework = await self._detect_framework() try: authored_agent = authored_agent_document( - bundle_dir / "contract.json", + (bundle_dir / "contract.json").read_text(encoding="utf-8"), subfolder=job.source.subfolder, framework=framework, ) diff --git a/tests/harness/test_agent_reading.py b/tests/harness/test_agent_reading.py index 48d654b68..1a218dfd7 100644 --- a/tests/harness/test_agent_reading.py +++ b/tests/harness/test_agent_reading.py @@ -13,6 +13,8 @@ import logging import os import shutil +import subprocess +import sys import threading import types import time @@ -123,7 +125,7 @@ async def send(self, message: str) -> None: async def interrupt(self) -> bool: self.backend.interrupts.append(time.monotonic()) if self.backend.interrupt_raises: - raise RuntimeError(self.backend.interrupt_says) + raise self.backend.interrupt_error(self.backend.interrupt_says) self.interrupted.set() return True @@ -146,6 +148,7 @@ def __init__(self, name: str, output: Path, turns: list) -> None: self.seen: list = [] self.interrupt_raises = False self.interrupt_says = "the interrupt control request timed out" + self.interrupt_error: type[BaseException] = RuntimeError self.start_raises = False self.saves_on_start: dict | None = None self.close_hangs = False @@ -590,18 +593,164 @@ async def saves_then_goes_quiet(live): assert any("did not close" in record.getMessage() for record in caplog.records) -def test_at_the_hard_limit_a_saved_contract_ends_as_the_reading_ends_it(tmp_path): - source = _repo(tmp_path) - job = HarnessJob.model_validate(_job()) +def test_a_cancel_during_the_stop_is_still_a_timeout(tmp_path, model, monkeypatch): + _repo(tmp_path) + _short_limit(monkeypatch) + fake = model(_quiet) + fake.interrupt_raises = True + fake.interrupt_error = asyncio.CancelledError output = tmp_path / "reading" - output.mkdir() - assert agent_reading._at_the_limit(job, source, output) is None - _save(output, CHAT) - assert agent_reading._at_the_limit(job, source, output) == 0 - assert (output / "agent.json").exists() - assert agent_reading._at_the_limit(job, source, output) == 0 - assert agent_reading.READ_AGENT.at_limit is agent_reading._at_the_limit + assert _run(tmp_path, wait=0.3) == 78 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_timed_out", False) + + +def test_an_invalid_saved_contract_logs_no_values_from_it(tmp_path, model, caplog): + _repo(tmp_path) + marker = "CUSTOMER-MARKER-7f3a db_password=hunter2" + model(_writes(text=json.dumps({**CHAT, "tools": marker}))) + output = tmp_path / "reading" + + with caplog.at_level("WARNING", logger=reading_result.__name__): + assert _run(tmp_path) == 78 + + assert _failure_in(output)["details"] == {"class": "ValidationError", "code": ""} + [logged] = [record for record in caplog.records if record.name == reading_result.__name__] + assert logged.exc_text is None + assert "hunter2" not in caplog.text + + +def test_the_contract_checked_is_the_contract_used(tmp_path, model, monkeypatch): + _repo(tmp_path) + model(_writes(CHAT)) + output = tmp_path / "reading" + real = reading_result.detect_framework + + def a_late_save_lands(*args, **kwargs): + # The session saves again while the scan runs: a contract over the 5 MiB cap. + (output / "contract.json").write_text( + json.dumps({**CHAT, "real_use_cases": ["x" * (6 * MIB)]}), encoding="utf-8" + ) + return real(*args, **kwargs) + + monkeypatch.setattr(reading_result, "detect_framework", a_late_save_lands) + + assert _run(tmp_path) == 0 + + agent = json.loads((output / "agent.json").read_text(encoding="utf-8"))["agent"] + assert agent == build_authored_agent( + AgentContract.model_validate(CHAT).model_dump(mode="json"), + framework=detect_framework(tmp_path / "source", repo_root=tmp_path / "source", package_dir=None), + ) + + +def test_an_environment_backed_job_is_read(tmp_path): + job = HarnessJob.model_validate( + {**_job(), "agent": {"connector": "vapi", "mode": "environment_backed", "config": {}}} + ) + + assert agent_reading._check_job(job) is None + + +_CLOSE_CHILD = r""" +import asyncio, dataclasses, json, sys, time +config = json.loads(sys.argv[1]) +sys.path[:0] = config["sys_path"] +from pathlib import Path +from fi.alk.harness import agent_reading, reading_entry, session +from fi.alk.harness.backends import StageDone +from fi.alk.harness.reading_entry import run_reading_mode + +reading_entry._SECRET_FILES = () +root = Path(config["root"]) +output = root / "reading" + + +class Live: + def __init__(self, spec): + self.spec = spec + + async def start(self): + pass + + async def send(self, message): + pass + + async def interrupt(self): + return True + + async def stop(self): + # A close that swallows every cancel, the loop's own shutdown's included. + while True: + try: + await asyncio.sleep(3600) + except asyncio.CancelledError: + pass + + async def replies(self): + (output / "contract.json").write_text(json.dumps(config["contract"])) + await asyncio.Event().wait() + yield StageDone(outcome="success", turns=1, cost_usd=0.0) + + +class Backend: + name = "vertex-gemini" + default_model = "gemini-2.5-flash" + + def create(self, spec): + return Live(spec) + + def can_drive(self, model): + return True + + +session.resolve = lambda *a, **k: Backend() +mode = dataclasses.replace( + agent_reading._read_agent(interrupt_wait=0.2), limit_margin_seconds=config["margin"] +) +(root / "started").write_text(repr(time.time())) +sys.exit(run_reading_mode(mode, job_path=root / "job.json", source=root / "source", output=output)) +""" + + +def test_a_close_that_never_ends_still_leaves_the_document_well_inside_the_hard_limit(tmp_path): + _repo(tmp_path) + (tmp_path / "job.json").write_text(json.dumps(_job()), encoding="utf-8") + soft, margin = 0.5, 6.0 + config = { + "sys_path": [entry for entry in sys.path if entry], + "root": str(tmp_path), + "contract": CHAT, + "margin": margin, + } + env = { + **os.environ, + _LIMIT_ENV: str(soft), + "ALK_HARNESS_MODEL": "gemini-2.5-flash", + "OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:9", + "OTEL_TRACES_EXPORTER": "none", + } + env.pop(spend.JOURNAL_ALIAS, None) + + finished = subprocess.run( + [sys.executable, "-c", _CLOSE_CHILD, json.dumps(config)], + env=env, + capture_output=True, + text=True, + timeout=60, + ) + + output = tmp_path / "reading" + started = float((tmp_path / "started").read_text(encoding="utf-8")) + assert finished.returncode == 0, finished.stderr + assert not (output / "failure.json").exists() + # Decided at soft + interrupt + wait + close (about 0.9 s), long before the hard limit + # (soft + margin); only the shutdown the close blocks is ended by the limit, with the code. + documented = (output / "agent.json").stat().st_mtime - started + assert documented < soft + margin - 3.0 + assert _cost_in(output)["complete"] is False def test_a_failed_document_is_logged_with_its_masked_trace(tmp_path, model, monkeypatch, caplog): @@ -636,7 +785,11 @@ def test_a_folder_mismatch_is_an_invalid_document(tmp_path, model): failure = _failure_in(output) assert (failure["code"], failure["retryable"]) == ("agent_document_invalid", False) # The build's own code for it, from the build's own document code. - assert failure["details"]["code"] == "agent_invalid" + assert failure["details"] == { + "class": "AgentBuildError", + "code": "agent_invalid", + "detail": "subfolder_mismatch", + } assert (output / "contract.json").exists() assert not (output / "agent.json").exists() _assert_cost(output, 0.11, True) diff --git a/tests/harness/test_reading_entry.py b/tests/harness/test_reading_entry.py index b274d7363..0924ad158 100644 --- a/tests/harness/test_reading_entry.py +++ b/tests/harness/test_reading_entry.py @@ -179,9 +179,6 @@ async def work(job, source, output): reading_entry.write_reading_cost(output, complete=True) if config["work"] == "completes_then_raises": raise RuntimeError("the work broke after its last write") - if config["work"] == "saves_then_blocks": - # Saved, then stuck where a session's close can hang: past the limit, never returning. - (output / "contract.json").write_text(json.dumps(config["contract"])) if config["work"] == "spoils_the_output_then_blocks": # Every write at the limit now fails, and each failure would log. shutil.rmtree(output) @@ -195,12 +192,7 @@ async def work(job, source, output): return 0 -extra = {} -if config.get("at_limit") == "read_agent": - from fi.alk.harness import agent_reading - - extra["at_limit"] = agent_reading._at_the_limit -mode = ReadingMode(check_job=lambda job: None, work=work, **config["mode"], **extra) +mode = ReadingMode(check_job=lambda job: None, work=work, **config["mode"]) (root / "started").write_text(repr(time.time())) code = run_reading_mode( mode, job_path=root / "job.json", source=root / "source", output=root / "reading" @@ -212,7 +204,7 @@ async def work(job, source, output): _BLOCK_SECONDS = 20.0 -def _run_in_a_child(root: Path, mode_values: dict, work: str, *, linger: float = 0.0, **more): +def _run_in_a_child(root: Path, mode_values: dict, work: str, *, linger: float = 0.0): """Returns the finished child and the seconds from the helper's start to the child's end.""" root.mkdir(parents=True, exist_ok=True) (root / "job.json").write_text(json.dumps(_job()), encoding="utf-8") @@ -223,7 +215,6 @@ def _run_in_a_child(root: Path, mode_values: dict, work: str, *, linger: float = "work": work, "block_seconds": _BLOCK_SECONDS, "linger_seconds": linger, - **more, } env = { **os.environ, @@ -511,28 +502,6 @@ def _empty_spend() -> dict: return {"schema": "futureagi.harness-spend.v1", "total_usd": 0.0, "unpriced_turns": 0, "stages": []} -def test_a_contract_saved_before_the_hard_limit_still_ends_with_its_document(tmp_path): - (tmp_path / "source").mkdir(parents=True) - contract = { - "agent": "support", - "modality": "chat", - "real_use_cases": ["order"], - "tools": [{"name": "add", "args": ["item_id"]}], - } - mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} - - finished, elapsed = _run_in_a_child( - tmp_path, mode_values, "saves_then_blocks", contract=contract, at_limit="read_agent" - ) - - output = tmp_path / "reading" - assert finished.returncode == 0, finished.stderr - assert 1.0 <= elapsed <= 1.0 + 5 - assert not (output / "failure.json").exists() - assert json.loads((output / "agent.json").read_text())["agent"] - assert _cost_in(output)["complete"] is False - - def test_a_timeout_after_a_complete_cost_marks_it_incomplete(tmp_path): mode_values = {**READ_AGENT_VALUES, "time_limit_seconds": 1.0} From 81d16b54f66d417261be8c589b60f537eed19c4f Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 14:17:13 +0530 Subject: [PATCH 17/23] fix(harness): mark a paid row short when a file is over the read size From the last review round on the fixes for the first review of this PR: - A file over 1 MiB in a model-found agent's folder now marks the row short (partial) instead of being skipped silently, so a paid row with a missing file is never reported complete. - The description logs one trace for a reader fault of an unforeseen kind, then counts the rest, as the walk does. - Comments say traces are logged with the platform's secret values masked, not that they are fully masked. TH-8950 Signed-off-by: khushalsonawat --- src/fi/alk/harness/scan_ai.py | 4 ++-- src/fi/alk/harness/scan_walk.py | 26 +++++++++++++++++++------- tests/harness/test_scan_walk.py | 22 ++++++++++++++++++++-- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/src/fi/alk/harness/scan_ai.py b/src/fi/alk/harness/scan_ai.py index b379fe808..d28b938f2 100644 --- a/src/fi/alk/harness/scan_ai.py +++ b/src/fi/alk/harness/scan_ai.py @@ -168,7 +168,7 @@ def _events(self, received: Any, turn: Turn) -> list[Any]: self._on_turn_end() except Exception: # A failed cost note must not turn a paid answer into a retried one. The - # trace is logged masked, so a fault in the cost writer can be found. + # trace is logged (secret values masked), so a fault in the cost writer can be found. logger.warning("find_agents: the cost note after the turn failed", exc_info=True) @@ -412,7 +412,7 @@ async def _answered(work: Callable[..., str], *args: Any) -> dict[str, Any]: except asyncio.TimeoutError: return _error(_TOO_SLOW) except Exception: - # The model is told only that the call failed; the trace is logged masked. + # The model is told only that the call failed; the trace is logged, secret values masked. logger.warning("find_agents: a tool call failed", exc_info=True) return _error(_FAILED) diff --git a/src/fi/alk/harness/scan_walk.py b/src/fi/alk/harness/scan_walk.py index 1ecc9af9d..a3c405e1d 100644 --- a/src/fi/alk/harness/scan_walk.py +++ b/src/fi/alk/harness/scan_walk.py @@ -845,7 +845,7 @@ def _take(self, directory: str, folder: str, name: str) -> None: raise except Exception: # A reader fault of a kind not foreseen is still this file's problem, never the - # walk's: the file is unread. The first trace is logged masked; the rest are + # walk's: the file is unread. The first trace is logged (secret values masked); the rest are # counted, so a fault that hits every file can't flood the log. if not self.faults: logger.warning("agent walk: a file's reader failed", exc_info=True) @@ -1260,6 +1260,8 @@ def __init__( # One file that couldn't be read or parsed: the counts are short, but the reading # goes on, as the walk goes on past an unread file. self.short = False + # Reader faults of a kind not foreseen; only the first one's trace is logged. + self.faults = 0 self.shadowed = _ShadowedNames(top, self.projects) def run(self) -> None: @@ -1283,8 +1285,7 @@ def run(self) -> None: except MemoryError: raise except Exception: - logger.warning("agent description: a template's reader failed", exc_info=True) - self.short = True + self._fault("a template's reader failed") start = os.path.join(self.top, self.subfolder) if self.subfolder else self.top for directory, folder, name in _source_files(self.top, start): if self._out_of_time(): @@ -1293,6 +1294,13 @@ def run(self) -> None: if self.partial: return + def _fault(self, what: str) -> None: + # A fault in the kit that hits every file logs one trace, not one per file. + if not self.faults: + logger.warning("agent description: %s", what, exc_info=True) + self.faults += 1 + self.short = True + def _out_of_time(self) -> bool: if self.clock() > self.deadline: self.partial = True @@ -1322,8 +1330,7 @@ def _take(self, directory: str, folder: str, name: str) -> None: except MemoryError: raise except Exception: - logger.warning("agent description: a file's reader failed", exc_info=True) - self.short = True + self._fault("a file's reader failed") def _read(self, path: str) -> bytes | None: """The file's bytes, or None for a file that can't or mustn't be read; sets @@ -1346,7 +1353,11 @@ def _read(self, path: str) -> bytes | None: if self.bytes_read > _DESCRIBE_BYTES: self.partial = True return None - return data if len(data) <= _FILE_BYTES else None + if len(data) > _FILE_BYTES: + # Over the size a file is read at: not read, so the counts are short. + self.short = True + return None + return data def describe_entry( @@ -1379,7 +1390,8 @@ def describe_entry( raise except Exception: # Runs after the model was paid for, so a fault in the kit keeps what was read and - # marks the row partial rather than ending in a paid retry. The trace is logged masked. + # marks the row partial rather than ending in a paid retry. The trace is logged, with the + # platform's secret values masked. logger.warning("agent description stopped", exc_info=True) reading.partial = True facts = reading.facts.get(entry) diff --git a/tests/harness/test_scan_walk.py b/tests/harness/test_scan_walk.py index 9d34cbd7a..316124ff5 100644 --- a/tests/harness/test_scan_walk.py +++ b/tests/harness/test_scan_walk.py @@ -1227,7 +1227,7 @@ def test_side_folders_cannot_use_up_the_file_cap_before_the_agent( assert result.stopped == "limit" -@pytest.mark.parametrize("shape", ["linked_template", "unterminated_py", "long_line_py"]) +@pytest.mark.parametrize("shape", ["linked_template", "unterminated_py", "long_line_py", "over_size_py"]) def test_one_unreadable_file_does_not_stop_an_ai_rows_description(tmp_path: Path, shape: str) -> None: """The row was paid for: a bad file marks it partial, and the reading goes on.""" _write( @@ -1242,8 +1242,10 @@ def test_one_unreadable_file_does_not_stop_an_ai_rows_description(tmp_path: Path (tmp_path / "app/.env.example").symlink_to(tmp_path / "outside.env") elif shape == "unterminated_py": (tmp_path / "app/a_gen.py").write_text("x = '''never closed\n", encoding="utf-8") - else: + elif shape == "long_line_py": (tmp_path / "app/a_gen.py").write_text("x = 1 # " + "y" * 70_000 + "\n", encoding="utf-8") + else: + (tmp_path / "app/a_gen.py").write_text("x = 1\n" * 300_000, encoding="utf-8") candidate = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) @@ -1314,3 +1316,19 @@ def broken(data: bytes, shadowed: object) -> object: assert result.unread_files == 5 assert caplog.text.count("a file's reader failed") == 1 assert "LookupError" in caplog.text + + +def test_a_reader_fault_on_every_described_file_logs_one_trace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + _write(tmp_path, {"app/bot.py": GRAPH, **{f"app/m{index}.py": "x = 1\n" for index in range(5)}}) + + def broken(data: bytes, shadowed: object) -> object: + raise LookupError("a fault in the kit's reader") + + monkeypatch.setattr(scan_walk, "_python_facts", broken) + with caplog.at_level("WARNING", logger=scan_walk.logger.name): + candidate = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) + + assert candidate is not None and candidate.partial is True + assert caplog.text.count("a file's reader failed") == 1 From b46311ad915509664800e2654030f8f807ecb0cd Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 14:27:46 +0530 Subject: [PATCH 18/23] fix(harness): cap the walk at 40,000 files so its memory stays in the sandbox From the last review round on the fixes for the first review of this PR: - The file cap is 40,000, not 200,000. What the walk holds grows with the names each file imports, not its bytes: about 33 KB a file for files that each import 200 names, a 1,389 MB peak at 40,000 such files. At 200,000 such files the sandbox would kill the scan. 40,000 still covers a repo whose own code passes 20,000 files. - The partial-row sentence says files couldn't be read, not that they weren't read in time, since a file that fails to parse also sets it. - A test pins that ALK_SCAN_AI_MAX_SECONDS is read when the model step is asked, and the comment says so. TH-8950 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_scan.py | 2 +- src/fi/alk/harness/scan_document.py | 2 +- src/fi/alk/harness/scan_walk.py | 7 ++++--- tests/harness/test_agent_scan.py | 9 +++++++++ tests/harness/test_scan_walk.py | 4 ++-- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/fi/alk/harness/agent_scan.py b/src/fi/alk/harness/agent_scan.py index e5cda067f..0446483dd 100644 --- a/src/fi/alk/harness/agent_scan.py +++ b/src/fi/alk/harness/agent_scan.py @@ -62,7 +62,7 @@ # hard stop, both when the reading starts. CLOCK = 1200.0 AI_ENV = "ALK_SCAN_AI_MAX_SECONDS" -# The model step's limit when AI_ENV doesn't name one; read when the scan starts, as the clock is. +# The model step's limit when AI_ENV doesn't name one; read when the model step is asked. AI_SECONDS = 600.0 # The helper's hard stop comes this long after the scan's own clock, so a scan that stops itself # always writes its own file first. diff --git a/src/fi/alk/harness/scan_document.py b/src/fi/alk/harness/scan_document.py index 961a989b5..7c38318b1 100644 --- a/src/fi/alk/harness/scan_document.py +++ b/src/fi/alk/harness/scan_document.py @@ -109,7 +109,7 @@ ), "agent_py_may_stop": "More than one agent.py sits under this folder, so a build may stop.", "not_finished": "The framework check for this folder did not finish.", - "partial": "Not every file under this folder was read in time, so counts may be short.", + "partial": "Not every file under this folder could be read, so counts may be short.", "found_by_ai": "Found by the AI step, not by the code check.", } _LONGEST_TEMPLATE = max(len(template) for template in _TEMPLATES.values()) diff --git a/src/fi/alk/harness/scan_walk.py b/src/fi/alk/harness/scan_walk.py index a3c405e1d..676c2b8b5 100644 --- a/src/fi/alk/harness/scan_walk.py +++ b/src/fi/alk/harness/scan_walk.py @@ -56,9 +56,10 @@ # A hundredth of a folder check's allowance: the crafted files that use a whole one # take seconds, so one file now costs about a second at most. _FILE_WORK = 1_000_000_000 -# Measured 10 Nov: 100,000 tiny source files walk in 13 s at 266 MB, so this sits above any -# real repo; the byte cap and the clock are what bound the walk. -_MAX_FILES = 200_000 +# What the walk holds grows with the names each file imports, not its bytes: about 33 KB a +# file for files that each import 200 names (measured 10 Nov, 1,389 MB peak at 40,000 such +# files), which leaves room for the folder check's child in a 4,096 MB sandbox. +_MAX_FILES = 40_000 _MAX_BYTES = 268_435_456 _NAMES_PER_FILE = 200 _IMPORT_DEPTH = 10 diff --git a/tests/harness/test_agent_scan.py b/tests/harness/test_agent_scan.py index 9965f1d0b..4c2c422ac 100644 --- a/tests/harness/test_agent_scan.py +++ b/tests/harness/test_agent_scan.py @@ -1211,3 +1211,12 @@ def test_every_scan_logger_masks_secrets() -> None: for module in (agent_scan, scan_walk, scan_folders, scan_ai): assert any(isinstance(f, MaskSecrets) for f in module.logger.filters), module.__name__ + + +def test_the_model_steps_limit_is_read_when_it_is_asked(tmp_path, monkeypatch, ai) -> None: + source = _write(tmp_path / "source", SIDE_ONLY) + monkeypatch.setenv(agent_scan.AI_ENV, "100") + + assert _scan(_job(), source, tmp_path / "out") == 0 + + assert [call["seconds"] for call in ai.calls] == [100.0] diff --git a/tests/harness/test_scan_walk.py b/tests/harness/test_scan_walk.py index 316124ff5..859f3481c 100644 --- a/tests/harness/test_scan_walk.py +++ b/tests/harness/test_scan_walk.py @@ -342,7 +342,7 @@ def test_a_broken_langgraph_json_is_unread_not_a_crash( def test_the_walk_limit_and_the_clock_stop_it( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - assert (scan_walk._MAX_FILES, scan_walk._MAX_BYTES) == (200_000, 268_435_456) + assert (scan_walk._MAX_FILES, scan_walk._MAX_BYTES) == (40_000, 268_435_456) _write(tmp_path, {"a.py": GRAPH, "b.py": GRAPH, "c.py": GRAPH}) calls: list[int] = [] @@ -361,7 +361,7 @@ def clock() -> float: assert by_count.stopped == "limit" assert _entries(by_count) == ["a.py", "b.py"] - monkeypatch.setattr(scan_walk, "_MAX_FILES", 200_000) + monkeypatch.setattr(scan_walk, "_MAX_FILES", 40_000) monkeypatch.setattr(scan_walk, "_MAX_BYTES", len(GRAPH)) by_bytes = _walk(tmp_path) assert by_bytes.stopped == "limit" From b8970348b3fc68837c0088a0f5dade898ffaa634 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 14:34:59 +0530 Subject: [PATCH 19/23] fix(harness): bound the step after the walk by the clock The step that joins the walk's files into agents (the import search from each candidate, and the per-agent loops) now checks a deadline: the walk's own deadline plus 120 s, well inside the helper's 600 s margin before the hard stop. Past it the walk ends as stopped by the clock, so the scan writes its own file (scan_timed_out, exit 0) instead of being ended by the hard stop. The two-pass test now pins the order it claims: a settings template in a folder that sorts after tests/ is read before tests/test_a.py. TH-8950 Signed-off-by: khushalsonawat --- src/fi/alk/harness/scan_walk.py | 45 +++++++++++++++++++++++++------- tests/harness/test_scan_walk.py | 46 ++++++++++++++++++++++++++++++--- 2 files changed, 78 insertions(+), 13 deletions(-) diff --git a/src/fi/alk/harness/scan_walk.py b/src/fi/alk/harness/scan_walk.py index 676c2b8b5..b58f509c9 100644 --- a/src/fi/alk/harness/scan_walk.py +++ b/src/fi/alk/harness/scan_walk.py @@ -199,6 +199,15 @@ class _Unread(Exception): pass +class _ResultOutOfTime(Exception): + """The step after the walk (joining files into agents) passed its deadline.""" + + +# The step after the walk may run this long past the walk's own deadline. It stays well inside +# the helper's margin before the hard stop (600 s), so a slow join ends in the scan's own file. +_RESULT_GRACE = 120.0 + + # What reading or parsing one file can raise: that file's problem, never the walk's. Anything # else is a fault in the kit and reaches the scan's crash path. _FILE_ERRORS = ( @@ -791,6 +800,8 @@ def __init__( self.unread = 0 # Unread files whose reader raised a kind not foreseen (a kit fault, not the file's). self.faults = 0 + # When the step after the walk must give up: see _RESULT_GRACE. + self.result_deadline = deadline + _RESULT_GRACE self.files_read = 0 self.bytes_read = 0 self.stopped: Literal["", "limit", "clock", "memory"] = "" @@ -929,6 +940,10 @@ def _imported(self, path: str) -> tuple[str, ...]: self._edges[path] = tuple(found) return self._edges[path] + def _in_time(self) -> None: + if self.clock() > self.result_deadline: + raise _ResultOutOfTime() + def _reached(self, start: str, candidates: dict[str, Sign]) -> list[str]: """The candidates ``start`` imports, through files that aren't candidates.""" reached: list[str] = [] @@ -937,6 +952,7 @@ def _reached(self, start: str, candidates: dict[str, Sign]) -> list[str]: for _ in range(_IMPORT_DEPTH): following: list[str] = [] for path in frontier: + self._in_time() for target in self._imported(path): if target in seen: continue @@ -988,6 +1004,7 @@ def agents(self, signs: dict[str, Sign]) -> list[_Agent]: ] kept = [] for entry in sorted(entries): + self._in_time() side = _is_side_path(entry) if signs[entry] in _WEAKEST: project = self._project_of(entry) @@ -998,10 +1015,11 @@ def agents(self, signs: dict[str, Sign]) -> list[_Agent]: ): continue kept.append((entry, side)) - return [ - _Agent(entry, signs[entry], side, self._subfolder(entry, side, kept)) - for entry, side in kept - ] + agents = [] + for entry, side in kept: + self._in_time() + agents.append(_Agent(entry, signs[entry], side, self._subfolder(entry, side, kept))) + return agents def _project_of(self, path: str) -> str: return self.projects.nearest(posixpath.dirname(path)) or "" @@ -1096,10 +1114,12 @@ def result(self, *, upload: bool) -> WalkResult: ) listed = agents[:1] if upload else agents[:_MAX_AGENTS] outside = {sign for path, sign in signs.items() if not _is_side_path(path)} + rows = [] + for agent in listed: + self._in_time() + rows.append(self.candidate(agent, listed, upload=upload)) return WalkResult( - agents=tuple( - self.candidate(agent, listed, upload=upload) for agent in listed - ), + agents=tuple(rows), more_than_fifty=not upload and len(agents) > _MAX_AGENTS, stopped=self.stopped, unread_files=self.unread, @@ -1123,7 +1143,7 @@ def _prompt_count(paths: list[str], subfolder: str) -> int: ) -def _empty_result(walk: _Walk, stopped: Literal["limit", "memory"]) -> WalkResult: +def _empty_result(walk: _Walk, stopped: Literal["limit", "memory", "clock"]) -> WalkResult: return WalkResult( agents=(), more_than_fifty=False, @@ -1161,7 +1181,14 @@ def walk_repo( return _empty_result(walk, "memory") # Any other fault here is the kit's, not the repository's: it goes to the scan's crash # path, never to "the repository is larger than the scan reads". - result = walk.result(upload=upload) + try: + result = walk.result(upload=upload) + except _ResultOutOfTime: + # Joining a huge import graph into agents ran past its deadline: the scan ends in its + # own file (scan_timed_out), never at the helper's hard stop. + logger.warning("agent walk: joining files into agents ran out of time") + walk.facts.clear() + return _empty_result(walk, "clock") logger.info( "agent walk: files_read=%d unread=%d reader_faults=%d agents=%d left_out=%d stopped=%s", walk.files_read, diff --git a/tests/harness/test_scan_walk.py b/tests/harness/test_scan_walk.py index 859f3481c..4b3b7da03 100644 --- a/tests/harness/test_scan_walk.py +++ b/tests/harness/test_scan_walk.py @@ -354,7 +354,8 @@ def clock() -> float: assert timed.stopped == "clock" assert _entries(timed) == ["a.py"] - assert len(calls) == 2 + # The walk checked before each of its two files; the join after it checks the clock too. + assert len(calls) >= 2 monkeypatch.setattr(scan_walk, "_MAX_FILES", 2) by_count = _walk(tmp_path) @@ -1269,6 +1270,7 @@ def test_every_file_is_read_in_exactly_one_pass(tmp_path: Path, monkeypatch: pyt "examples/demo/agent.py": GRAPH, "pkg/sub/mod.py": "x = 1\n", "pkg/sub/tests/test_b.py": "x = 1\n", + "zz_app/.env.example": "C=1\n", }, ) seen: list[str] = [] @@ -1284,9 +1286,10 @@ def counting(self, directory, folder, name): every = sorted(p.relative_to(tmp_path).as_posix() for p in tmp_path.rglob("*") if p.is_file()) assert sorted(seen) == every assert len(seen) == len(set(seen)) - # The customer's own code and its settings template come before any side file. - own = seen[: seen.index("examples/demo/agent.py")] - assert "app/.env.example" in own and "agent.py" in own + # A settings template is read with its folder's own code, before every side file: here + # one in a folder that sorts after tests/, which a side-pass template would follow. + assert seen.index("zz_app/.env.example") < seen.index("tests/test_a.py") + assert seen.index("app/.env.example") < seen.index("app/test_helpers.py") def test_a_template_beside_the_agent_is_read_before_a_big_tests_folder( @@ -1332,3 +1335,38 @@ def broken(data: bytes, shadowed: object) -> object: assert candidate is not None and candidate.partial is True assert caplog.text.count("a file's reader failed") == 1 + + +def test_the_step_after_the_walk_watches_the_clock(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Dileep's shape, small: many files that build a client, each importing a web of modules. + Joining them into agents must stop at its deadline and end as a clock stop, never run on + to the helper's hard stop.""" + files = { + f"pkg/m{index}.py": "".join(f"from .m{(index * 7 + step) % 200} import x\n" for step in range(1, 6)) + for index in range(200) + } + files.update( + { + f"pkg/c{index}.py": CLIENT + "".join(f"from .m{(index * 3 + step) % 200} import x\n" for step in range(5)) + for index in range(50) + } + ) + _write(tmp_path, files) + + complete = _walk(tmp_path) + assert complete.stopped == "" and complete.agents + + monkeypatch.setattr(scan_walk, "_RESULT_GRACE", 0.0) + calls: list[int] = [] + + def clock() -> float: + calls.append(1) + return float(len(calls)) + + # Each file read checks the clock once, so this deadline lets the walk finish and then + # gives the join only a few checks. + timed = walk_repo(tmp_path, deadline=float(len(files) + 5), clock=clock) + + assert timed.stopped == "clock" + assert timed.agents == () + assert len(calls) < len(files) + 50 From 49c1f4c60ec74068f4f715b5344ffef64759aa9f Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 15:21:44 +0530 Subject: [PATCH 20/23] fix(harness): leave the document room after the longest stop, and keep a password out of a failed redaction - The hard limit is soft + 300 s, so the document after the longest stop has room on a slow CPU; a test pins the stop's bounds inside it. - The interrupt runs as its own task: one that swallows its cancel can't hold the stop, and Ctrl-C passes through. - A URL the parser refuses is redacted whole instead of raising with its credentials in the message. - The folder choice uses the contract text already checked. TH-8951 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_reading.py | 30 +++-- src/fi/alk/harness/framework_detection.py | 17 ++- src/fi/alk/harness/reading_result.py | 13 ++- src/fi/alk/providers/agent.py | 6 +- tests/harness/test_agent_reading.py | 129 ++++++++++++++++++---- tests/harness/test_reading_entry.py | 2 +- 6 files changed, 158 insertions(+), 39 deletions(-) diff --git a/src/fi/alk/harness/agent_reading.py b/src/fi/alk/harness/agent_reading.py index 64c0607ad..359f86d9e 100644 --- a/src/fi/alk/harness/agent_reading.py +++ b/src/fi/alk/harness/agent_reading.py @@ -55,14 +55,15 @@ _LIMIT_ENV = "ALK_READING_MAX_DURATION_SECONDS" _DEFAULT_SOFT_LIMIT_SECONDS = 1800.0 -# The hard limit outlasts the soft one by the interrupt, the wait after it, and a margin, so a -# reading that stops itself is never cut off by the helper first. -_HARD_LIMIT_MARGIN_SECONDS = 180.0 +# The hard limit outlasts the soft one by the longest stop (the interrupt, the wait after it and +# the close: 90 s) and room for the saved contract's document after it, whose framework scan can +# take well over a minute on a crafted repository, so a reading that stops itself is never cut +# off by the helper first. +_HARD_LIMIT_MARGIN_SECONDS = 300.0 _INTERRUPT_SECONDS = 10.0 _INTERRUPT_WAIT_SECONDS = 60.0 -# The SDK's own close gives up after about 20 s. With it, the decision comes by soft + 90 s, which -# leaves the saved contract's document (its framework scan, about 70 s at the scanner's cap) room -# to finish inside the hard limit's margin. +# Ours, not the SDK's: its close is shielded and has no bound of its own. With it the decision +# comes by soft + 90 s, which leaves the rest of the margin to the document. _CLOSE_SECONDS = 20.0 _COUNTS = ("tokens_in", "tokens_out", "tokens_cached") @@ -183,11 +184,20 @@ async def _stop( ) -> None: # Set first, so a turn ending now doesn't earn the reminder. stopping.set() - try: - await asyncio.wait_for(stage.interrupt_response(), _INTERRUPT_SECONDS) - except (Exception, asyncio.CancelledError): + # Its own task, waited on with a bound: wait_for would wait again for a cancel the interrupt + # swallows, and catching the cancel around it would swallow a Ctrl-C too. + interrupting = asyncio.ensure_future(stage.interrupt_response()) + bound = min(wait, _INTERRUPT_SECONDS) + await asyncio.wait({interrupting}, timeout=bound) + if not interrupting.done(): + interrupting.cancel() + logger.warning("the stop at the time limit was not delivered within %g s", bound) + elif interrupting.cancelled() or interrupting.exception() is not None: # A stop that can't be delivered is still a timeout, not a crash. - logger.warning("the stop at the time limit was not delivered", exc_info=True) + logger.warning( + "the stop at the time limit was not delivered", + exc_info=None if interrupting.cancelled() else interrupting.exception(), + ) await asyncio.wait({task}, timeout=wait) # Closed before the files are read, so a late tool call is unlikely to save after the # decision. The close is bounded, though: a session that hangs closing must not keep the diff --git a/src/fi/alk/harness/framework_detection.py b/src/fi/alk/harness/framework_detection.py index 8c6705b0f..508aa366f 100644 --- a/src/fi/alk/harness/framework_detection.py +++ b/src/fi/alk/harness/framework_detection.py @@ -969,11 +969,15 @@ def detect_framework( return {"name": "unknown", "version": None} -def _contract_workdir(source: Path, bundle_dir: Path) -> Path | None: +def _contract_workdir( + source: Path, bundle_dir: Path, contract_text: str | None = None +) -> Path | None: """`runtime.workdir` from the sealed contract, when it names a real inner folder.""" try: contract = json.loads( - (bundle_dir / "contract.json").read_text(encoding="utf-8") + contract_text + if contract_text is not None + else (bundle_dir / "contract.json").read_text(encoding="utf-8") ) runtime = contract.get("runtime") if isinstance(contract, dict) else None workdir = runtime.get("workdir") if isinstance(runtime, dict) else None @@ -989,12 +993,15 @@ def _contract_workdir(source: Path, bundle_dir: Path) -> Path | None: def named_framework_folder( - source: Path, bundle_dir: Path, subfolder: str | None + source: Path, bundle_dir: Path, subfolder: str | None, *, contract_text: str | None = None ) -> Path | None: - """The folder the job names, else the sealed contract's own; ``None`` when neither does.""" + """The folder the job names, else the sealed contract's own; ``None`` when neither does. + + ``contract_text`` is the contract already read, when the caller has read it. + """ if subfolder: return agent_folder(source, subfolder) - return _contract_workdir(source, bundle_dir) + return _contract_workdir(source, bundle_dir, contract_text) def framework_folders_for( diff --git a/src/fi/alk/harness/reading_result.py b/src/fi/alk/harness/reading_result.py index 496f4e29a..0927f59a4 100644 --- a/src/fi/alk/harness/reading_result.py +++ b/src/fi/alk/harness/reading_result.py @@ -170,9 +170,14 @@ def __init__(self, sizes: dict[str, int]) -> None: self.sizes = sizes -def _framework_folder(job: HarnessJob, source: Path, output: Path) -> Path: +def _framework_folder(job: HarnessJob, source: Path, output: Path, contract_text: str) -> Path: # As a build chooses it when there is no build: the named folder, else the contract's own. - return named_framework_folder(source, output, job.source.subfolder) or source + return ( + named_framework_folder( + source, output, job.source.subfolder, contract_text=contract_text + ) + or source + ) def _write_whole(path: Path, body: bytes) -> None: @@ -202,7 +207,9 @@ def document(job: HarnessJob, source: Path, output: Path) -> int: raise _TooLarge({"contract_bytes": contract_bytes}) contract_text = raw.decode("utf-8") framework = detect_framework( - _framework_folder(job, source, output), repo_root=source, package_dir=None + _framework_folder(job, source, output, contract_text), + repo_root=source, + package_dir=None, ) # The build's own document, made by the build's own code. made = authored_agent_document( diff --git a/src/fi/alk/providers/agent.py b/src/fi/alk/providers/agent.py index e0cb43529..b0ad7f1ce 100644 --- a/src/fi/alk/providers/agent.py +++ b/src/fi/alk/providers/agent.py @@ -245,7 +245,11 @@ def _redact(value: Any, key: str = "") -> Any: if isinstance(value, list): return [_redact(item, key) for item in value] if isinstance(value, str) and "://" in value: - parsed = urlsplit(value) + try: + parsed = urlsplit(value) + except ValueError: + # A netloc urlsplit refuses may still hold credentials, so none of it is kept. + return _REDACTED if parsed.scheme and parsed.netloc: hostname = parsed.hostname or "" try: diff --git a/tests/harness/test_agent_reading.py b/tests/harness/test_agent_reading.py index 1a218dfd7..84bce491d 100644 --- a/tests/harness/test_agent_reading.py +++ b/tests/harness/test_agent_reading.py @@ -13,6 +13,7 @@ import logging import os import shutil +import signal import subprocess import sys import threading @@ -623,26 +624,42 @@ def test_an_invalid_saved_contract_logs_no_values_from_it(tmp_path, model, caplo def test_the_contract_checked_is_the_contract_used(tmp_path, model, monkeypatch): - _repo(tmp_path) - model(_writes(CHAT)) + source = _repo(tmp_path) + (source / "agents/graph").mkdir(parents=True) + (source / "agents/graph/graph.py").write_text( + "from langgraph.graph import StateGraph\nStateGraph(int)\n", encoding="utf-8" + ) + checked = {**CHAT, "runtime": {"language": "python", "workdir": SUPPORT}} + model(_writes(checked)) output = tmp_path / "reading" - real = reading_result.detect_framework + real = reading_result.named_framework_folder def a_late_save_lands(*args, **kwargs): - # The session saves again while the scan runs: a contract over the 5 MiB cap. + # The session saves again before the folder is chosen: another folder, over the cap. (output / "contract.json").write_text( - json.dumps({**CHAT, "real_use_cases": ["x" * (6 * MIB)]}), encoding="utf-8" + json.dumps( + { + **CHAT, + "runtime": {"language": "python", "workdir": "agents/graph"}, + "real_use_cases": ["x" * (6 * MIB)], + } + ), + encoding="utf-8", ) return real(*args, **kwargs) - monkeypatch.setattr(reading_result, "detect_framework", a_late_save_lands) + monkeypatch.setattr(reading_result, "named_framework_folder", a_late_save_lands) assert _run(tmp_path) == 0 agent = json.loads((output / "agent.json").read_text(encoding="utf-8"))["agent"] + # The checked contract's folder and framework, not the late one's. assert agent == build_authored_agent( - AgentContract.model_validate(CHAT).model_dump(mode="json"), - framework=detect_framework(tmp_path / "source", repo_root=tmp_path / "source", package_dir=None), + AgentContract.model_validate(checked).model_dump(mode="json"), + framework=detect_framework(source / SUPPORT, repo_root=source, package_dir=None), + ) + assert detect_framework(source / "agents/graph", repo_root=source, package_dir=None) != ( + detect_framework(source / SUPPORT, repo_root=source, package_dir=None) ) @@ -655,7 +672,7 @@ def test_an_environment_backed_job_is_read(tmp_path): _CLOSE_CHILD = r""" -import asyncio, dataclasses, json, sys, time +import asyncio, dataclasses, json, os, signal, sys, time config = json.loads(sys.argv[1]) sys.path[:0] = config["sys_path"] from pathlib import Path @@ -679,9 +696,22 @@ async def send(self, message): pass async def interrupt(self): + if config["interrupt"] == "swallows": + # Swallows the first cancel, then hangs on; only the loop's shutdown ends it. + try: + await asyncio.sleep(3600) + except asyncio.CancelledError: + pass + await asyncio.sleep(3600) + if config["interrupt"] == "sigint": + # A Ctrl-C that lands while the stop is being delivered. + os.kill(os.getpid(), signal.SIGINT) + await asyncio.sleep(3600) return True async def stop(self): + if config["close"] != "hangs": + return # A close that swallows every cancel, the loop's own shutdown's included. while True: try: @@ -690,7 +720,8 @@ async def stop(self): pass async def replies(self): - (output / "contract.json").write_text(json.dumps(config["contract"])) + if config["contract"] is not None: + (output / "contract.json").write_text(json.dumps(config["contract"])) await asyncio.Event().wait() yield StageDone(outcome="success", turns=1, cost_usd=0.0) @@ -715,15 +746,17 @@ def can_drive(self, model): """ -def test_a_close_that_never_ends_still_leaves_the_document_well_inside_the_hard_limit(tmp_path): - _repo(tmp_path) - (tmp_path / "job.json").write_text(json.dumps(_job()), encoding="utf-8") - soft, margin = 0.5, 6.0 +def _read_in_a_child(root: Path, *, soft: float, margin: float, **behaviour): + _repo(root) + (root / "job.json").write_text(json.dumps(_job()), encoding="utf-8") config = { "sys_path": [entry for entry in sys.path if entry], - "root": str(tmp_path), + "root": str(root), "contract": CHAT, "margin": margin, + "interrupt": "honours", + "close": "ends", + **behaviour, } env = { **os.environ, @@ -733,7 +766,6 @@ def test_a_close_that_never_ends_still_leaves_the_document_well_inside_the_hard_ "OTEL_TRACES_EXPORTER": "none", } env.pop(spend.JOURNAL_ALIAS, None) - finished = subprocess.run( [sys.executable, "-c", _CLOSE_CHILD, json.dumps(config)], env=env, @@ -741,9 +773,15 @@ def test_a_close_that_never_ends_still_leaves_the_document_well_inside_the_hard_ text=True, timeout=60, ) + return finished, float((root / "started").read_text(encoding="utf-8")) + + +def test_a_close_that_never_ends_still_leaves_the_document_well_inside_the_hard_limit(tmp_path): + soft, margin = 0.5, 6.0 + + finished, started = _read_in_a_child(tmp_path, soft=soft, margin=margin, close="hangs") output = tmp_path / "reading" - started = float((tmp_path / "started").read_text(encoding="utf-8")) assert finished.returncode == 0, finished.stderr assert not (output / "failure.json").exists() # Decided at soft + interrupt + wait + close (about 0.9 s), long before the hard limit @@ -753,6 +791,59 @@ def test_a_close_that_never_ends_still_leaves_the_document_well_inside_the_hard_ assert _cost_in(output)["complete"] is False +def test_an_interrupt_that_swallows_its_cancel_does_not_hold_the_reading(tmp_path): + soft, margin = 0.5, 6.0 + + finished, started = _read_in_a_child(tmp_path, soft=soft, margin=margin, interrupt="swallows") + + output = tmp_path / "reading" + assert finished.returncode == 0, finished.stderr + assert not (output / "failure.json").exists() + assert (output / "agent.json").stat().st_mtime - started < soft + margin - 3.0 + + +def test_a_ctrl_c_during_the_stop_passes_through(tmp_path): + finished, _ = _read_in_a_child( + tmp_path, soft=0.5, margin=6.0, interrupt="sigint", contract=None + ) + + output = tmp_path / "reading" + assert finished.returncode in (-signal.SIGINT, 130), finished.stderr + assert "KeyboardInterrupt" in finished.stderr + # Not a timeout: an interrupt ends the reading as an interrupt, before any save. + assert not (output / "failure.json").exists() + assert _cost_in(output)["complete"] is False + + +def test_the_longest_stop_leaves_room_for_the_document_inside_the_margin(): + # The stop's own bounds, then the saved contract's document, whose framework scan can take + # over a minute on a crafted repository: about 150 s is kept for it. + stop = ( + agent_reading._INTERRUPT_SECONDS + + agent_reading._INTERRUPT_WAIT_SECONDS + + agent_reading._CLOSE_SECONDS + ) + assert stop + 150.0 < agent_reading._HARD_LIMIT_MARGIN_SECONDS + assert agent_reading.READ_AGENT.limit_margin_seconds == agent_reading._HARD_LIMIT_MARGIN_SECONDS + + +def test_a_url_the_redaction_cannot_parse_leaves_no_password_anywhere( + tmp_path, model, capfd, caplog +): + _repo(tmp_path) + leaky = "https://admin:hunter2\uff03x@db.example/" + model(_writes({**CHAT, "real_use_cases": [leaky]})) + output = tmp_path / "reading" + + with caplog.at_level("DEBUG"): + assert _run(tmp_path) == 0 + + kept = (output / "agent.json").read_text(encoding="utf-8") + captured = capfd.readouterr() + assert not (output / "failure.json").exists() + assert "hunter2" not in kept + captured.out + captured.err + caplog.text + + def test_a_failed_document_is_logged_with_its_masked_trace(tmp_path, model, monkeypatch, caplog): secret = "sk-document-0123456789" secrets = tmp_path / "secrets.json" @@ -1949,7 +2040,7 @@ def test_a_bad_time_limit_falls_back_to_the_default(monkeypatch, value): monkeypatch.setenv(_LIMIT_ENV, value) assert agent_reading._soft_limit() == 1800.0 - assert reading_entry._hard_limit(agent_reading.READ_AGENT) == 1800.0 + 180.0 + assert reading_entry._hard_limit(agent_reading.READ_AGENT) == 1800.0 + 300.0 def test_a_good_time_limit_is_read(monkeypatch): @@ -1958,7 +2049,7 @@ def test_a_good_time_limit_is_read(monkeypatch): assert agent_reading._soft_limit() == 120.0 mode = agent_reading.READ_AGENT # The helper's limit is the reading's own plus its margin, and a timeout repeats if retried. - assert reading_entry._hard_limit(mode) == 120.0 + 180.0 + assert reading_entry._hard_limit(mode) == 120.0 + 300.0 assert mode.timed_out_retryable is False # The contract's codes, as the platform reads them. assert (mode.timed_out_code, mode.crashed_code) == ( diff --git a/tests/harness/test_reading_entry.py b/tests/harness/test_reading_entry.py index 0924ad158..a5a062076 100644 --- a/tests/harness/test_reading_entry.py +++ b/tests/harness/test_reading_entry.py @@ -27,7 +27,7 @@ READ_AGENT_VALUES = { "name": "read_agent", - "time_limit_seconds": 1980.0, + "time_limit_seconds": 2100.0, "timed_out_code": "agent_reading_timed_out", "timed_out_retryable": False, "crashed_code": "agent_reading_crashed", From edb692636d0b5cbf4d26cf9ce542fa024fab3e2f Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 15:24:04 +0530 Subject: [PATCH 21/23] test(harness): a browser reading gets its one look before the refusal #224's rule now gives every browser reading one second look first. TH-8951 Signed-off-by: khushalsonawat --- tests/harness/test_agent_reading.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/harness/test_agent_reading.py b/tests/harness/test_agent_reading.py index 84bce491d..67c6fc71d 100644 --- a/tests/harness/test_agent_reading.py +++ b/tests/harness/test_agent_reading.py @@ -403,7 +403,8 @@ def test_a_browser_agent_is_refused_by_the_real_gate(tmp_path, model): assert _run(tmp_path, _job(subfolder=SCRAPER)) == 78 assert _failure_in(output)["code"] == "agent_type_unsupported" - assert fake.submissions == 1 + # One second look before any refusal: the browser answer to it stands and is refused. + assert fake.submissions == 2 assert len(fake.sent) == 1 assert not (output / "agent.json").exists() assert not (output / "contract.json").exists() From af8e86e573b32077c7d8d40f0123a6101f0a0a03 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 15:35:34 +0530 Subject: [PATCH 22/23] fix(harness): cap the import pairs the step after the walk holds Candidates that reach each other hold one entry per pair, which grows with the square of the candidates. The join now counts them as it builds them and, past 5,000,000 (about 90 MB), ends as the walk's other caps do (stopped="limit", scan_limit_reached), instead of growing until the clock stops it. TH-8950 Signed-off-by: khushalsonawat --- src/fi/alk/harness/scan_walk.py | 28 +++++++++++++++++++++------- tests/harness/test_scan_walk.py | 21 +++++++++++++++++++++ 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/src/fi/alk/harness/scan_walk.py b/src/fi/alk/harness/scan_walk.py index b58f509c9..68c5532bf 100644 --- a/src/fi/alk/harness/scan_walk.py +++ b/src/fi/alk/harness/scan_walk.py @@ -203,6 +203,14 @@ class _ResultOutOfTime(Exception): """The step after the walk (joining files into agents) passed its deadline.""" +class _ResultTooBig(Exception): + """The step after the walk found more candidate-to-candidate imports than it holds.""" + + +# Candidates that reach each other hold one entry each per pair, which grows with the square of +# the candidates. About 17.5 bytes an entry (Dileep's measurement, 10 Nov), so this is ~90 MB. +_MAX_EDGES = 5_000_000 + # The step after the walk may run this long past the walk's own deadline. It stays well inside # the helper's margin before the hard stop (600 s), so a slow join ends in the scan's own file. _RESULT_GRACE = 120.0 @@ -973,14 +981,15 @@ def _absorbs(self, importer: str, imported: str, signs: dict[str, Sign]) -> bool def agents(self, signs: dict[str, Sign]) -> list[_Agent]: nodes = sorted(signs) - edges = { - node: [ - target - for target in self._reached(node, signs) - if self._absorbs(node, target, signs) + edges: dict[str, list[str]] = {} + held = 0 + for node in nodes: + edges[node] = [ + target for target in self._reached(node, signs) if self._absorbs(node, target, signs) ] - for node in nodes - } + held += len(edges[node]) + if held > _MAX_EDGES: + raise _ResultTooBig() component_of: dict[str, int] = {} components = _components(nodes, edges) for number, members in enumerate(components): @@ -1189,6 +1198,11 @@ def walk_repo( logger.warning("agent walk: joining files into agents ran out of time") walk.facts.clear() return _empty_result(walk, "clock") + except _ResultTooBig: + # Too many candidates reach each other to hold: the same end as the walk's other caps. + logger.warning("agent walk: joining files into agents passed its edge budget") + walk.facts.clear() + return _empty_result(walk, "limit") logger.info( "agent walk: files_read=%d unread=%d reader_faults=%d agents=%d left_out=%d stopped=%s", walk.files_read, diff --git a/tests/harness/test_scan_walk.py b/tests/harness/test_scan_walk.py index 4b3b7da03..6231c446f 100644 --- a/tests/harness/test_scan_walk.py +++ b/tests/harness/test_scan_walk.py @@ -1370,3 +1370,24 @@ def clock() -> float: assert timed.stopped == "clock" assert timed.agents == () assert len(calls) < len(files) + 50 + + +def test_the_step_after_the_walk_holds_a_bounded_number_of_edges( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Dileep's shape, small: every candidate imports one package that re-exports modules, so + each candidate reaches every other and the pairs grow with the square of the candidates.""" + files = {"pkg/__init__.py": "".join(f"from .m{index} import x\n" for index in range(20))} + files.update({f"pkg/m{index}.py": "x = 1\n" for index in range(20)}) + files.update({f"c{index}.py": CLIENT + "import pkg\nfrom pkg import x\n" for index in range(30)}) + files.update({f"pkg/m{index}.py": CLIENT for index in range(20, 40)}) + files["pkg/__init__.py"] += "".join(f"from .m{index} import x\n" for index in range(20, 40)) + _write(tmp_path, files) + + assert _walk(tmp_path).stopped == "" + + monkeypatch.setattr(scan_walk, "_MAX_EDGES", 10) + bounded = _walk(tmp_path) + + assert bounded.stopped == "limit" + assert bounded.agents == () From d1f968cc28c760509aa71bbc0196ce3c9a8cc47d Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 10 Nov 2026 16:35:27 +0530 Subject: [PATCH 23/23] fix(reading): answer Dileep's third review of #245 - A Ctrl-C after the contract is saved now passes through; a cancel leaking from the session still keeps the saved contract. - A BaseExceptionGroup (or any non-interrupt, non-exit base exception) from the work ends as the crash failure with exit 1. - _redact logs only the error class when it redacts a URL whole. - The stop-budget test pins exactly 90 s. - Drop a person's name from a scan_walk.py comment. Refs: TH-8951 Signed-off-by: khushalsonawat --- src/fi/alk/harness/agent_reading.py | 10 +++- src/fi/alk/harness/reading_entry.py | 7 ++- src/fi/alk/harness/scan_walk.py | 2 +- src/fi/alk/providers/agent.py | 9 +++- tests/harness/test_agent_reading.py | 53 ++++++++++++++++--- .../test_scenario_source_provision_agent.py | 19 +++++++ 6 files changed, 86 insertions(+), 14 deletions(-) diff --git a/src/fi/alk/harness/agent_reading.py b/src/fi/alk/harness/agent_reading.py index 359f86d9e..7ce74e0b5 100644 --- a/src/fi/alk/harness/agent_reading.py +++ b/src/fi/alk/harness/agent_reading.py @@ -185,7 +185,7 @@ async def _stop( # Set first, so a turn ending now doesn't earn the reminder. stopping.set() # Its own task, waited on with a bound: wait_for would wait again for a cancel the interrupt - # swallows, and catching the cancel around it would swallow a Ctrl-C too. + # swallows. A Ctrl-C here reaches `_read`, which lets it through. interrupting = asyncio.ensure_future(stage.interrupt_response()) bound = min(wait, _INTERRUPT_SECONDS) await asyncio.wait({interrupting}, timeout=bound) @@ -325,7 +325,13 @@ async def _read(job: HarnessJob, source: Path, output: Path, *, interrupt_wait: except (Exception, asyncio.CancelledError) as exc: # A contract the gate saved is paid for; a crash would have the platform pay again. # A cancel leaking out of the session is the same: the saved contract still stands. - if saved_file(output) != CONTRACT_FILE: + # A cancel of this task itself (a Ctrl-C) is let through, the saved files as they are. + # (3.10 has no `cancelling`, and there a Ctrl-C is raised as itself, not as a cancel.) + current = asyncio.current_task() + if saved_file(output) != CONTRACT_FILE or ( + isinstance(exc, asyncio.CancelledError) + and getattr(current, "cancelling", lambda: 0)() + ): raise logger.warning( "reading fault after a saved contract: class=%s", type(exc).__name__, exc_info=True diff --git a/src/fi/alk/harness/reading_entry.py b/src/fi/alk/harness/reading_entry.py index 810ae820f..b94c74c7d 100644 --- a/src/fi/alk/harness/reading_entry.py +++ b/src/fi/alk/harness/reading_entry.py @@ -365,8 +365,11 @@ def run_reading_mode(mode: ReadingMode, *, job_path: Path, source: Path, output: try: logger.info("reading started at %s", datetime.now(timezone.utc).isoformat()) code = asyncio.run(_work_then_stop_the_clock(mode, job, source, output, limit)) - except (Exception, asyncio.CancelledError) as exc: - # A cancel leaking out of the work is a crash too; an interrupt or exit is let through. + except BaseException as exc: + # A cancel or a group of them leaking out of the work is a crash too; an interrupt or an + # exit is let through. + if isinstance(exc, (KeyboardInterrupt, SystemExit)): + raise logger.error("the %s reading crashed", mode.name, exc_info=True) crash = exc finally: diff --git a/src/fi/alk/harness/scan_walk.py b/src/fi/alk/harness/scan_walk.py index 68c5532bf..3077c7b61 100644 --- a/src/fi/alk/harness/scan_walk.py +++ b/src/fi/alk/harness/scan_walk.py @@ -208,7 +208,7 @@ class _ResultTooBig(Exception): # Candidates that reach each other hold one entry each per pair, which grows with the square of -# the candidates. About 17.5 bytes an entry (Dileep's measurement, 10 Nov), so this is ~90 MB. +# the candidates. About 17.5 bytes an entry, so this is ~90 MB. _MAX_EDGES = 5_000_000 # The step after the walk may run this long past the walk's own deadline. It stays well inside diff --git a/src/fi/alk/providers/agent.py b/src/fi/alk/providers/agent.py index b0ad7f1ce..18cf6dbdd 100644 --- a/src/fi/alk/providers/agent.py +++ b/src/fi/alk/providers/agent.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging from collections.abc import Mapping, Sequence from typing import Any, Literal from urllib.parse import urlsplit, urlunsplit @@ -8,6 +9,8 @@ from .setup_hash import compute_setup_hash +logger = logging.getLogger(__name__) + AGENT_SCHEMA_VERSION = "1.0" MAIN_UNIT_ID = "main" @@ -247,8 +250,10 @@ def _redact(value: Any, key: str = "") -> Any: if isinstance(value, str) and "://" in value: try: parsed = urlsplit(value) - except ValueError: - # A netloc urlsplit refuses may still hold credentials, so none of it is kept. + except ValueError as refused: + # A netloc urlsplit refuses may still hold credentials, so none of it is kept, and + # only the error's class is logged: its text repeats the netloc. + logger.warning("a URL was redacted whole: class=%s", type(refused).__name__) return _REDACTED if parsed.scheme and parsed.netloc: hostname = parsed.hostname or "" diff --git a/tests/harness/test_agent_reading.py b/tests/harness/test_agent_reading.py index 67c6fc71d..6cb89b154 100644 --- a/tests/harness/test_agent_reading.py +++ b/tests/harness/test_agent_reading.py @@ -8,6 +8,7 @@ import argparse import asyncio +import builtins import dataclasses import json import logging @@ -105,6 +106,8 @@ def __init__(self, backend: "_Backend", spec) -> None: async def start(self) -> None: if self.backend.saves_on_start is not None: _save(self.backend.output, self.backend.saves_on_start) + if self.backend.start_error is not None: + raise self.backend.start_error if self.backend.start_raises: raise FileNotFoundError("the Claude CLI was not found") @@ -153,6 +156,7 @@ def __init__(self, name: str, output: Path, turns: list) -> None: self.start_raises = False self.saves_on_start: dict | None = None self.close_hangs = False + self.start_error: BaseException | None = None self.submissions = 0 def create(self, spec) -> _Live: @@ -723,6 +727,9 @@ async def stop(self): async def replies(self): if config["contract"] is not None: (output / "contract.json").write_text(json.dumps(config["contract"])) + if config.get("after_save") == "sigint": + # A Ctrl-C just after the save, before the reading has decided anything. + os.kill(os.getpid(), signal.SIGINT) await asyncio.Event().wait() yield StageDone(outcome="success", turns=1, cost_usd=0.0) @@ -816,15 +823,47 @@ def test_a_ctrl_c_during_the_stop_passes_through(tmp_path): assert _cost_in(output)["complete"] is False +def test_a_ctrl_c_after_the_save_passes_through_and_leaves_the_files(tmp_path): + finished, _ = _read_in_a_child(tmp_path, soft=5.0, margin=6.0, after_save="sigint") + + output = tmp_path / "reading" + assert finished.returncode in (-signal.SIGINT, 130), finished.stderr + assert "KeyboardInterrupt" in finished.stderr + # The saved contract is left as it is; nothing is written over it or after it. + assert (output / "contract.json").exists() + assert not (output / "failure.json").exists() + assert not (output / "agent.json").exists() + + +@pytest.mark.skipif(sys.version_info < (3, 11), reason="exception groups arrived in 3.11") +def test_a_group_of_cancels_before_the_session_opens_is_a_crash_with_its_file(tmp_path, model): + _repo(tmp_path) + fake = model() + group = getattr(builtins, "BaseExceptionGroup") + # The shape an anyio task group gives when a child task is cancelled. + fake.start_error = group("the session's tasks failed", [asyncio.CancelledError()]) + output = tmp_path / "reading" + + assert _run(tmp_path) == 1 + + failure = _failure_in(output) + assert (failure["code"], failure["retryable"]) == ("agent_reading_crashed", True) + assert failure["message"].startswith("BaseExceptionGroup") + _assert_cost(output, 0.0, False) + + def test_the_longest_stop_leaves_room_for_the_document_inside_the_margin(): - # The stop's own bounds, then the saved contract's document, whose framework scan can take - # over a minute on a crafted repository: about 150 s is kept for it. - stop = ( - agent_reading._INTERRUPT_SECONDS - + agent_reading._INTERRUPT_WAIT_SECONDS - + agent_reading._CLOSE_SECONDS + # The stop's own bounds, exactly: the interrupt, the wait after it, and the close. + bounds = ( + agent_reading._INTERRUPT_SECONDS, + agent_reading._INTERRUPT_WAIT_SECONDS, + agent_reading._CLOSE_SECONDS, ) - assert stop + 150.0 < agent_reading._HARD_LIMIT_MARGIN_SECONDS + assert bounds == (10.0, 60.0, 20.0) + assert sum(bounds) == 90.0 + # Then the saved contract's document, whose framework scan can take over a minute on a + # crafted repository: at least 150 s of the margin stays for it. + assert sum(bounds) + 150.0 < agent_reading._HARD_LIMIT_MARGIN_SECONDS assert agent_reading.READ_AGENT.limit_margin_seconds == agent_reading._HARD_LIMIT_MARGIN_SECONDS diff --git a/tests/harness/test_scenario_source_provision_agent.py b/tests/harness/test_scenario_source_provision_agent.py index 9cf5e53a9..49a4fc21e 100644 --- a/tests/harness/test_scenario_source_provision_agent.py +++ b/tests/harness/test_scenario_source_provision_agent.py @@ -126,6 +126,25 @@ def test_provision_payload_carries_the_agent_built_from_the_sealed_contract( ) +def test_a_url_urlsplit_refuses_is_registered_redacted_and_logged_by_class_only( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + # A full-width number sign NFKC-normalises to "#", so urlsplit refuses the netloc, and its + # error repeats the whole userinfo. + leaky = "https://admin:hunter2\uff03x@db.example/" + _write_bundle(tmp_path, _contract(leaky)) + + with caplog.at_level(logging.DEBUG): + payload = _provision(tmp_path, SourceKind.GITHUB) + + assert payload["agent"]["agent"]["behaviour"][0]["instructions"] == "[redacted]" + assert "hunter2" not in json.dumps(payload["agent"]) + [logged] = [record for record in caplog.records if record.name == "fi.alk.providers.agent"] + assert logged.getMessage() == "a URL was redacted whole: class=ValueError" + assert logged.exc_info is None + assert "hunter2" not in caplog.text + + @pytest.mark.parametrize( ("kind", "contract", "build_error", "logged"), [