diff --git a/.github/workflows/sdk-smoke.yml b/.github/workflows/sdk-smoke.yml index 8589e0e0d..5544706f2 100644 --- a/.github/workflows/sdk-smoke.yml +++ b/.github/workflows/sdk-smoke.yml @@ -100,6 +100,9 @@ jobs: tests/harness/test_reading_entry.py tests/harness/test_agent_reading.py tests/harness/test_scan_walk.py + tests/harness/test_browser_scan_cut.py + tests/harness/test_save_child.py + tests/harness/test_save_checker.py tests/harness/test_scan_folders.py tests/harness/test_scan_document.py tests/harness/test_scan_ai.py diff --git a/src/fi/alk/harness/HOW-IT-WORKS.md b/src/fi/alk/harness/HOW-IT-WORKS.md index a267bf261..28fb4aa48 100644 --- a/src/fi/alk/harness/HOW-IT-WORKS.md +++ b/src/fi/alk/harness/HOW-IT-WORKS.md @@ -80,7 +80,8 @@ conversation**, so the model corrects and resubmits rather than a bad contract l A browser reading gets one second look first, naming any browser, crawling or computer-use tool the code check found in the source. So does a chat or voice reading of source -where the check found one. The next submission stands: a browser answer is refused to +where the check found one, or where the check stopped before reading all of it. The next +submission stands: a browser answer is refused to `failure.json` and the build exits 78, and a chat or voice answer is saved. The look gives a value to send back, which only tells apart calls in one model message: those run before the model reads any result. The rule and its state are `SecondLook` in `agent_type_check.py`. A browser answer to a chat or voice look is refused at once, since that look said browser stops the build. When calls in one message run one after another, or together once the first scan is done, a later call is taken as the answer before the model has read the look. Once an answer with the value has come back, a later browser answer without it gets a look again, so answers that keep switching cost turns, up to the reading's turn limit (70, or 140 for a generic job read unattended), never a refusal without a look. diff --git a/src/fi/alk/harness/agent_type_check.py b/src/fi/alk/harness/agent_type_check.py index e6e467d6c..b1814574d 100644 --- a/src/fi/alk/harness/agent_type_check.py +++ b/src/fi/alk/harness/agent_type_check.py @@ -15,6 +15,7 @@ import os import re import tokenize +from collections import Counter from collections.abc import Iterable, Iterator, Sequence from functools import lru_cache from pathlib import Path @@ -25,10 +26,12 @@ from .instrumentation_evidence import _SKIPPED_DIRECTORIES, _imports from .job import FailureDomain, HarnessFailure, HarnessStage from .python_source import ( + _RAW_SAFE_CODECS, LARGEST_PARSED_MODULE, EscapeBudget, ParseBudget, TokenBudget, + _coding, parsed, read_module, ) @@ -118,6 +121,7 @@ _MARKER_START = re.compile( r"computer[_-]?[uU]se|ComputerUse|computer_\d{8}|@playwright/mcp|mcp-server-playwright" ) +_MARKER_START_BYTES = re.compile(_MARKER_START.pattern.encode()) _TOOLKIT = "PlayWrightBrowserToolkit" # A whole name: no letter, digit or "_" before or after it. In a unicode-escape file read as bytes, # an escape such as \t, \x20, \040 or \u0020 just before it counts as a boundary too. @@ -143,6 +147,16 @@ _LARGEST_CODE_FILE = 1024 * 1024 # What one scan searches with regular expressions in files it can't decode (see _tools_in). _LARGEST_RAW_SEARCH = 1024 * 1024 +# A file too big to read is searched as bytes for the names that would make it matter (see +# _may_name_a_tool), up to this size: a generated or data module of a few MB is cleared. Larger +# ones aren't read at all. +_LARGEST_UNREAD_SEARCHED = 4 * 1024 * 1024 +# And what one scan searches of them in all. Every name is looked for in each file: about 18 ms a +# MiB on 3.12.3, crafted near misses included, so this much is about 0.15 s. A coding line naming a +# codec costs more: the lookup normalises the name in Python, so two crafted 4 MiB names took +# 0.31 s, and encodings._cache keeps each name it misses, 8 M characters for those two, for as long +# as the process runs. read_module looks up the names of files it reads the same way. +_UNREAD_SEARCH_TOTAL = 8 * 1024 * 1024 # Past this a repository holds the conversation up for longer than the rest of the scan is worth. _LARGEST_SCAN = 128 * 1024 * 1024 # Scripts are read by regular expressions that call back for every string or comment: 128 MB of @@ -400,13 +414,24 @@ def _agent_files( yield Path(path) -def _packages_used(path: Path) -> set[str]: +def _packages_used(path: Path) -> set[str] | None: + """The browser packages ``path`` lists, or None when it can't be read: npm may still read it. + + Read as Node reads it, a byte that isn't UTF-8 becoming a replacement character. Text that + isn't JSON lists nothing: npm refuses it too. + """ try: + # A link to nothing or a pipe isn't a manifest npm reads either. if not path.is_file(): return set() # Windows editors often save a BOM, which json refuses. - manifest = json.loads(path.read_text(encoding="utf-8").removeprefix("\ufeff")) - except (OSError, ValueError, RecursionError): + text = path.read_text(encoding="utf-8", errors="replace").removeprefix("\ufeff") + # JSON.parse reads an integer of any length, which json refuses past 4,300 digits; the + # scan uses no numbers, so they're kept as text. + manifest = json.loads(text, parse_int=str) + except (OSError, RecursionError): + return None + except ValueError: return set() if not isinstance(manifest, dict): return set() @@ -423,13 +448,14 @@ def _packages_used(path: Path) -> set[str]: return found -def _imports_in(text: str, path: Path) -> set[str]: +def _imports_in(text: str, path: Path) -> set[str] | None: + """The modules ``text`` imports, or None when it won't parse: then they're unknown, not none.""" # One file at a time: a customer's odd file can run the parser out of memory, and that mustn't # take the rest of the scan, or the reading it serves, down with it. tree = parsed(text) if tree is None: logger.debug("agent type scan read no imports from %s", path) - return set() + return None return _imports(tree) @@ -471,14 +497,15 @@ def _script_kept(hit: re.Match[str]) -> str: return kept -def _python_markers(text: str, tokens: TokenBudget, path: Path) -> set[str]: +def _python_markers(text: str, tokens: TokenBudget, path: Path) -> set[str] | None: + """The tool markers in ``text``'s code, or None when the scan's tokenizing ran out first.""" if not _MARKER_START.search(text): return set() if not tokens.spend(text): # Past the total, comments can't be told from code, and a marker in one would refuse a # browser reading without its second look; so the file's markers aren't read, and the - # model's reading stands. - return set() + # scan says it was cut. + return None try: found = tokens.tokens(text) except (ValueError, SyntaxError, SystemError, tokenize.TokenError) as error: @@ -486,7 +513,7 @@ def _python_markers(text: str, tokens: TokenBudget, path: Path) -> set[str]: logger.debug("agent type scan read no markers from %s: %s", path, type(error).__name__) return set() # Past the work, too, the file's markers aren't read. - return set() if found is None else _markers(_python_code(found)) + return None if found is None else _markers(_python_code(found)) def _markers(code: str) -> set[str]: @@ -516,6 +543,46 @@ def _file_size(path: Path) -> int: return 0 +def _may_name_a_tool(path: Path, size: int, left: list[int]) -> bool: + """Whether a file too big to read, of ``size`` bytes, may name a tool, so leaving it unread + cuts the scan. + + Its bytes are searched for the names a Python file must hold to be read at all + (``_PYTHON_NAMES``, as ``read_module`` searches them), "omputer" aside: that is a word in + prose, so it counts only where a ``_MARKER_START`` match starts. Every such match holds + "omputer" or "playwright", so this covers a script's tool markers too. It's never decoded or + parsed. A Python file whose coding line names a codec not proven safe to search as bytes can + spell a name its bytes don't show, so it may; so may one whose coding line Python refuses, + since the codec can be registered at run time before the file is imported. A file that can't + be cleared may: one over ``_LARGEST_UNREAD_SEARCHED``, over what is ``left`` of the scan's + ``_UNREAD_SEARCH_TOTAL``, one that can't be read, or a path that isn't a regular file. So may + a ``package.json``: the packages it lists aren't among these names. + """ + if path.name == "package.json": + return True + if size > _LARGEST_UNREAD_SEARCHED or size > left[0]: + return True + try: + # A link can name a device or a pipe, which reads without end. + if not path.is_file(): + return True + with path.open("rb") as handle: + raw = handle.read(_LARGEST_UNREAD_SEARCHED + 1) + except (OSError, ValueError): + return True + left[0] -= len(raw) + # It may have grown since its size was taken. + if len(raw) > _LARGEST_UNREAD_SEARCHED or left[0] < 0: + return True + if path.suffix == ".py": + coding = _coding(raw) + if coding is None or coding[1].name not in _RAW_SAFE_CODECS: + return True + if any(name in raw for name in _PYTHON_NAMES if name != b"omputer"): + return True + return b"omputer" in raw and _MARKER_START_BYTES.search(raw) is not None + + def browser_tools_found(source_root: Path | None) -> list[str]: """The browser, crawling and computer-use tools the agent's own code uses, by display name. @@ -525,7 +592,26 @@ def browser_tools_found(source_root: Path | None) -> list[str]: if source_root is None: return [] root = Path(source_root).resolve() - return list(_scan(root, _signature(root))) + return list(_scan(root, _signature(root))[0]) + + +def browser_scan( + source_root: Path | None, subfolder: str | None = None, reached: list[Path] | None = None +) -> tuple[list[str], frozenset[str]]: + """What ``browser_tools_found``, or with ``subfolder`` ``browser_tools_in_agent``, finds, and + the limits that cut the scan (see ``_tools_in``), from one walk: a second walk would cost the + conversation the scan's time again. + + Raises ``SubfolderNotFound`` for a folder that can't be used. + """ + if source_root is None: + return [], frozenset() + if not subfolder: + root = Path(source_root).resolve() + names, cut = _scan(root, _signature(root)) + else: + names, cut = _scanned_agent(source_root, subfolder, reached) + return list(names), cut def _is_read(path: Path) -> bool: @@ -566,7 +652,9 @@ def _signature(root: Path) -> tuple[int, int, int]: @lru_cache(maxsize=8) -def _scan(root: Path, _files: tuple[int, int, int]) -> tuple[str, ...]: +def _scan( + root: Path, _files: tuple[int, int, int] +) -> tuple[tuple[str, ...], frozenset[str]]: """One scan per state of the files: every builder reopen in a process asks again.""" # Each record read once: the walk keeps the top's rows for the modules there. left = [_LARGEST_RECORDS] @@ -580,41 +668,61 @@ def _scan(root: Path, _files: tuple[int, int, int]) -> tuple[str, ...]: def _tools_in( paths: Iterable[Path], local: set[str], first: Path | None = None -) -> tuple[str, ...]: +) -> tuple[tuple[str, ...], frozenset[str]]: """The browser tools ``paths`` import or name, the files in ``first`` read before the rest: - the totals below cut the files read last, and the agent's own mustn't be those.""" + the totals below cut the files read last, and the agent's own mustn't be those. + + And the limits that cut the scan, by name: each left code unread or unparsed, so finding + nothing then doesn't say there's nothing, and the reading gets a second look. A file that + names no tool, or that Python can't decode, isn't one: it can't import what Python runs. A + file too big to read is searched as bytes to say whether it names one (``_may_name_a_tool``), + so a large data module that doesn't isn't one either. + """ found: set[str] = set() imported: set[str] = set() + # A count, not a flag: the log says how much each limit left unread. + cut: Counter[str] = Counter() scanned = scripts = python = 0 budget = ParseBudget() tokens = TokenBudget() escapes = EscapeBudget() searched = ParseBudget(_LARGEST_RAW_SEARCH) - skipped = 0 - for path in sorted( + unread = [_UNREAD_SEARCH_TOTAL] + ordered = sorted( filter(_is_read, paths), key=lambda path: (first is not None and not is_within(path, first), *_reading_order(path)), - ): + ) + for at, path in enumerate(ordered): size = _file_size(path) - # Never read, so it mustn't use up the limit before the code that is. + # Never read as code, so it mustn't use up the limits before the code that is: what its + # search reads has its own total. if size > _largest_read(path): + if _may_name_a_tool(path, size, unread): + cut["file_size"] += 1 continue scanned += size if scanned > _LARGEST_SCAN: + cut["scan_total"] += len(ordered) - at break if path.suffix in _SCRIPT_SUFFIXES: scripts += size if scripts > _LARGEST_SCRIPT_SCAN: + cut["script_total"] += 1 continue elif path.suffix == ".py": python += size if python > _LARGEST_PYTHON_SCAN: + cut["python_total"] += 1 continue # The tokenizer's known faults are caught in `_python_markers`; this catch is for the ones # nobody foresaw. try: if path.name == "package.json": - found |= _packages_used(path) + packages = _packages_used(path) + if packages is None: + cut["package_unreadable"] += 1 + else: + found |= packages elif path.suffix != ".py": found |= _markers_in(path) elif ( @@ -623,9 +731,16 @@ def _tools_in( ) ) is not None: markers = _python_markers(text, tokens, path) - names = _imports_in(text, path) if budget.spend(text) else set() + if markers is None: + cut["token_total"] += 1 + markers = set() found |= markers - imported |= names + if not budget.spend(text): + cut["parse_total"] += 1 + elif (names := _imports_in(text, path)) is None: + cut["parse_failed"] += 1 + else: + imported |= names elif undecoded: # A codec refused by name, which Python still runs the file in: the browser # packages and tool markers it names, as bytes, count. That leads to a second @@ -641,18 +756,29 @@ def _tools_in( # Whole names only: "steelers" in a string is no import of steel. named = {name for name in named if _WHOLE_NAMES[name].search(raw)} found |= _markers(raw) + else: + # A tool marker it names then goes unseen. + cut["search_total"] += 1 imported |= named except Exception as error: # A skipped file may be the agent's main one, and the reading then finds nothing. No # trace at any level, since a trace can quote a line of the customer's code; the type - # only at WARNING, once a scan, since a path can name their business. - skipped += 1 - if skipped == 1: + # only at WARNING, once a scan, since a path can name their business. The count is in + # the cut line below. + cut["skipped"] += 1 + if cut["skipped"] == 1: logger.warning("agent type scan skipped a file: %s", type(error).__name__) logger.debug("agent type scan skipped %s: %s", path, type(error).__name__) continue - if skipped > 1: - logger.warning("agent type scan skipped %d files", skipped) + if escapes.left < 0: + # A file past the total was searched as bytes, which can miss a name its codec spells. + cut["decode_total"] = 1 + if cut: + # Names and counts only: a path can name the customer's business. + logger.warning( + "agent type scan cut by %s", + ", ".join(f"{name} {count}" for name, count in sorted(cut.items())), + ) for name in imported: top = name.split(".")[0] if top in local: @@ -661,7 +787,7 @@ def _tools_in( found.add(top) elif name.endswith(f".{_TOOLKIT}"): found.add(_TOOLKIT) - return tuple(sorted({name.replace("_", "-") for name in found})) + return tuple(sorted({name.replace("_", "-") for name in found})), frozenset(cut) def _files_in(folder: Path) -> list[Path]: @@ -732,6 +858,14 @@ def browser_tools_in_agent( Raises ``SubfolderNotFound`` for a folder that can't be used. """ + return list(_scanned_agent(source_root, subfolder, reached)[0]) + + +def _scanned_agent( + source_root: Path, subfolder: str, reached: list[Path] | None +) -> tuple[tuple[str, ...], frozenset[str]]: + """``browser_tools_in_agent``'s scan, and the limits that cut it. The walk's own stops aren't + among them: they're logged by ``_walked``.""" root = Path(source_root).resolve() folder = agent_folder(root, subfolder) above = [ @@ -800,12 +934,8 @@ def unpacked(path: Path) -> bool: for parent in path.parents if is_within(parent, root) and (parent / "package.json").is_file() ] - return list( - _tools_in( - dict.fromkeys((*_agent_files(folder), *above, *shared, *manifests)), - local, - folder, - ) + return _tools_in( + dict.fromkeys((*_agent_files(folder), *above, *shared, *manifests)), local, folder ) @@ -822,18 +952,30 @@ def is_browser(text: str) -> bool: def decide( - claimed: Any, found: Sequence[str], *, looked_again: bool + claimed: Any, + found: Sequence[str], + *, + looked_again: bool, + cut: frozenset[str] = frozenset(), ) -> Literal["refuse", "second_look", "keep"]: """What the model read against what the code check found. A browser reading always gets one - more look first, and so does a reading the check disagrees with; the answer then stands.""" + more look first, and so does a reading the check disagrees with, or one it couldn't check all + of (``cut``, the limits that stopped it); the answer then stands.""" if is_browser(modality_text(claimed)): # Never at once: what was found can be a dev dependency or a helper script of a chat # agent, and a refusal can't be retried. return "refuse" if looked_again else "second_look" - return "second_look" if found and not looked_again else "keep" + # A cut is never a refusal: a limit reached on an honest repository says nothing of its job. + return "second_look" if (found or cut) and not looked_again else "keep" SECOND_LOOK_READ = "second_look_read" +# Shared by both cut texts, which must say the same of the check. +_CHECK_STOPPED = ( + "the code check stopped before reading all of the agent's code (it holds more than the check " + "reads, or a file it couldn't read), so it can't say whether a browser, crawling or " + "computer-use tool is used" +) class SecondLook: @@ -859,7 +1001,13 @@ def given(self) -> bool: return bool(self._receipts) def verdict( - self, claimed: str, receipt: Any, found: Sequence[str], *, given_before: bool + self, + claimed: str, + receipt: Any, + found: Sequence[str], + *, + given_before: bool, + cut: frozenset[str] = frozenset(), ) -> Literal["refuse", "second_look", "keep"]: # A call holding a value read a look, so its answer stands. Without one, the call after a # look that no value came back for is taken as the answer. Once a value came back, a call @@ -874,19 +1022,28 @@ def verdict( claimed, found, looked_again=read_it or (given_before and (self._unanswered or same_answer)), + cut=cut, ) - def issue(self, claimed: str, found: Sequence[str], receipt: str) -> str: + def issue( + self, + claimed: str, + found: Sequence[str], + receipt: str, + *, + cut: frozenset[str] = frozenset(), + ) -> str: """The look's text, with ``receipt`` as the value to send back.""" self._receipts.add(receipt) self._unanswered = True return ( - f"{self.prompt(claimed, found)} When you submit again, set " + f"{self.prompt(claimed, found, cut=cut)} When you submit again, set " f'{SECOND_LOOK_READ} to "{receipt}", which shows you read this.' ) @staticmethod - def prompt(claimed: str, found: Sequence[str]) -> str: + def prompt(claimed: str, found: Sequence[str], *, cut: frozenset[str] = frozenset()) -> str: + # What was found already says what to decide, cut or not, so its text is the same. if is_browser(claimed) and found: return ( "second-look: you recorded modality browser, which stops this build as not " @@ -896,6 +1053,16 @@ def prompt(claimed: str, found: Sequence[str]) -> str: "For a conversation, submit again as chat or voice. If its main job is that work, " "submit again as browser. Decide this from the source; don't ask anyone." ) + if is_browser(claimed) and cut: + # Nothing found in code left unread isn't nothing found. + return ( + "second-look: you recorded modality browser, which stops this build as not " + f"supported, and {_CHECK_STOPPED}. A chat or voice agent can use one in its tests " + "or a helper script. Does a person come to this agent for a conversation, or to " + "have it carry out work on websites or a computer? For a conversation, submit " + "again as chat or voice. If you're sure, submit again as browser. Decide this " + "from the source; don't ask anyone." + ) if is_browser(claimed): return ( "second-look: you recorded modality browser, which stops this build as not " @@ -905,6 +1072,15 @@ def prompt(claimed: str, found: Sequence[str]) -> str: "voice. If you're sure, submit again as browser. Decide this from the source; " "don't ask anyone." ) + if cut and not found: + return ( + f"second-look: you recorded modality {claimed or 'chat'}, and {_CHECK_STOPPED}. " + "A chat or voice agent can use one in its tests or a helper script. If a person " + "comes to this agent for a conversation, submit again as is. If its main job is " + "carrying out work on websites or a computer, submit again as browser, which " + "stops this build as not supported. Decide this from the source; don't ask " + "anyone." + ) return ( f"second-look: you recorded modality {claimed or 'chat'}, and the source uses " f"{', '.join(found)}. If that's a helper and a person comes to this agent for a " diff --git a/src/fi/alk/harness/forkserver_preload.py b/src/fi/alk/harness/forkserver_preload.py new file mode 100644 index 000000000..97b945f8a --- /dev/null +++ b/src/fi/alk/harness/forkserver_preload.py @@ -0,0 +1,19 @@ +"""The modules the kit's fork server loads once, for every child it starts. + +`multiprocessing` keeps one fork server per process, and the last `set_forkserver_preload` call +before it starts is the one that counts. The folder checks and the save checks share it, so both +set this one list, and whichever starts first preloads what the other needs too. Module names +only: importing them here would load the kit into every process that reads the list. +""" + +PRELOAD = [ + "fi.alk.harness.framework_detection", + "fi.alk.harness.tools", + "fi.alk.harness.bundle_author_v2", + "fi.alk.harness.source_tool_evidence", + "fi.alk.harness.instrumentation_evidence", + "fi.alk.harness.source_folder", + "fi.alk.harness.model_citation", + "fi.alk.harness.runtime_repair", + "fi.alk.harness.save_child", +] diff --git a/src/fi/alk/harness/save_checker.py b/src/fi/alk/harness/save_checker.py new file mode 100644 index 000000000..f4ef4ca99 --- /dev/null +++ b/src/fi/alk/harness/save_checker.py @@ -0,0 +1,1184 @@ +"""Runs the contract save's checks of the customer's files in a child process, under one deadline. + +The checks parse the customer's code, and Python's parser holds the GIL, so in a thread a crafted +repository holds the whole conversation for as long as they run. A child can be killed when a +save's deadline passes; a thread can't. A save the deadline cuts saves nothing. When no child can +be had, the same checks run in a thread, as they did before there was a child. + +The child answers two jobs: the session job (the browser scan and instrumentation, which depend +only on the session) once, cached here; and the save job once per save. Every limit inside the +checks stays: they decide the answers and bound memory, and the deadline only stops what they +don't. +""" + +from __future__ import annotations + +import asyncio +import itertools +import logging +import multiprocessing +import os +import queue +import subprocess +import sys +import threading +import time +import weakref +from collections.abc import Callable +from dataclasses import dataclass, field +from multiprocessing.connection import Connection +from multiprocessing.process import BaseProcess +from pathlib import Path +from typing import Any, Literal + +from .contract import AgentContract +from .forkserver_preload import PRELOAD +from .reading_entry import MaskSecrets + +logger = logging.getLogger(__name__) +logger.addFilter(MaskSecrets()) + +Mode = Literal["child", "thread"] + +# The whole of one save's wait, from the call's start: the session job, the lock, the child's +# start-up and its own checks. +SAVE_DEADLINE = 1.5 +# The session job's own bound. No save waits on it past its own deadline, so this only decides +# how long a crafted repository answers "not checked in time". +SESSION_BOUND = 30.0 +# From the child's `up`, so the fork server's start-up isn't charged to the trial. +TRIAL_SECONDS = 2.0 +# A killed child is gone almost at once; the bound only stops a wedged reap holding a thread. +JOIN_SECONDS = 5.0 +# A child that sends no `up` by then is taken as a failed start: the fork server's preload took +# 0.3-1.6 s when measured, so this only catches one that never comes up. +START_SECONDS = 15.0 +# Deaths in a row before the checks move into this process for the rest of the checker's life. +DEATHS = 2 +WATCH_INTERVAL = 0.05 +# Measured child peaks were 57-209 MB; this catches slow growth, not a burst. +WATCH_CAP = 400 * 1024 * 1024 + +STAGES = ("contract", "tool_evidence", "prompt_copy", "model", "runtime") +_PARTS = ("scan", "instrumentation") +_SCAN_LATE = "not checked in time: the folder's checks are still running; submit it once, alone" +_SAVE_LATE = "not checked in time: submit it once, alone" +_GONE: dict[str, Any] = {"id": None, "kind": "gone"} + +# Every checker started in this process, so the tests can close what they leave open. +_OPEN: weakref.WeakSet[SaveChecker] = weakref.WeakSet() + + +class SessionChecksFailed(Exception): + """A part of the session job raised in the child; the call fails and nothing is saved.""" + + def __init__(self, part: str, error: str, message: str) -> None: + super().__init__( + f"The folder's {part} check raised {error} in the save checker's child: {message}. " + "Nothing was saved; the next save runs it again." + ) + self.part, self.error, self.message = part, error, message + + +class SaveChecksFailed(Exception): + """The save job raised in the child; the call fails and nothing is saved.""" + + def __init__(self, error: str, message: str) -> None: + super().__init__( + f"The contract's checks raised {error} in the save checker's child: {message}. " + "Nothing was saved." + ) + self.error, self.message = error, message + + +class StandInFailed(Exception): + """The stand-in module named for the child's checks could not be put in place.""" + + def __init__(self, module: str, detail: str) -> None: + super().__init__( + f"The save checks' stand-in {module!r} could not be put in place in the child: " + f"{detail}. Check that the module imports and that its STAGES keys name real " + "attributes." + ) + self.module, self.detail = module, detail + + +@dataclass(frozen=True) +class ScanAnswer: + found: list[str] + cut: frozenset[str] + + +@dataclass(frozen=True) +class Saved: + """Every check answered; the caller writes it.""" + + contract: AgentContract + + +@dataclass(frozen=True) +class Refused: + """The response to send back, as `_problems` builds it.""" + + response: dict[str, Any] + + +@dataclass(frozen=True) +class NotInTime: + """Cut with no problem in; nothing is saved.""" + + response: dict[str, Any] + + +async def _by(awaitable: Any, deadline: float) -> Any: + """``awaitable``'s result, or ``asyncio.TimeoutError`` once the running loop's clock passes + ``deadline``. On 3.11 and later through ``asyncio.timeout_at``: there ``wait_for`` returns a + result that arrived in the same tick as a cancel, and the cancelled call would go on to save.""" + if sys.version_info >= (3, 11): + async with asyncio.timeout_at(deadline): + return await awaitable + return await asyncio.wait_for(awaitable, max(deadline - asyncio.get_running_loop().time(), 0.0)) + + +def _log_failed(part: str, message: dict[str, Any]) -> None: + # The type alone at ERROR: the message and trace can quote the customer's paths, which + # masking doesn't hide, so they go to DEBUG. + logger.error("save checker: %s failed in the child: %s", part, message.get("error")) + logger.debug( + "save checker: %s failed in the child: %s\n%s", part, message.get("message"), message.get("trace") + ) + + +def default_mode() -> Mode: + """"thread" when ALK_SAVE_CHECKS == "thread" or forkserver isn't available; else "child".""" + if os.environ.get("ALK_SAVE_CHECKS") == "thread": + return "thread" + if "forkserver" not in multiprocessing.get_all_start_methods(): + return "thread" + return "child" + + +def rss_bytes(source: Literal["statm", "ps"], text: str, page_size: int) -> int: + """A process's resident memory in bytes, from one raw reading. + + ``statm``'s second field counts resident pages; ``ps -o rss=`` prints KiB. Raises + ``ValueError`` on a reading it can't parse, which the watchdog skips. + """ + fields = text.split() + try: + if source == "statm": + return int(fields[1]) * page_size + if source == "ps": + return int(fields[0]) * 1024 + except (IndexError, ValueError) as unreadable: + raise ValueError(f"unreadable {source} reading") from unreadable + raise ValueError(f"unknown memory reading source {source!r}") + + +def _rss(pid: int) -> int | None: + try: + if sys.platform.startswith("linux"): + text = Path(f"/proc/{pid}/statm").read_text(encoding="ascii") + return rss_bytes("statm", text, os.sysconf("SC_PAGE_SIZE")) + out = subprocess.run( + ["ps", "-o", "rss=", "-p", str(pid)], + capture_output=True, + text=True, + timeout=1.0, + check=True, + ).stdout + return rss_bytes("ps", out, 0) + except (OSError, ValueError, subprocess.SubprocessError): + # A child that just ended, or a busy `ps`: the next tick reads again. + return None + + +def _late(text: str) -> dict[str, Any]: + return {"content": [{"type": "text", "text": text}], "is_error": True} + + +def _which(parts: list[str]) -> str: + if len(parts) == 2: + return "the scan and instrumentation" + return "the scan" if parts == ["scan"] else "instrumentation" + + +def _text(path: Path | None) -> str | None: + return None if path is None else os.fspath(path) + + +def _settle(future: asyncio.Future[None]) -> None: + if not future.done(): + future.set_result(None) + + +class _Thread: + """Marks a wait that must give way to thread mode.""" + + +_THREAD = _Thread() + + +@dataclass(eq=False) +class _Child: + process: BaseProcess + conn: Connection + pid: int + # `up`, and the start thread's trial and stand_in answers. + inbox: queue.SimpleQueue[dict[str, Any]] = field(default_factory=queue.SimpleQueue) + replies: dict[int, Callable[[dict[str, Any]], None]] = field(default_factory=dict) + # The id of the request in the child; the watchdog reads only while there is one. + request: int | None = None + ready: bool = False + # Why the parent killed it; None when it ended on its own. + killed: str | None = None + gone: bool = False + handled: bool = False + reaped: bool = False + reader: threading.Thread | None = None + stop: threading.Event = field(default_factory=threading.Event) + + +@dataclass(eq=False) +class _Session: + id: int + child: _Child + pending: set[str] + timer: threading.Timer | None = None + + +class SaveChecker: + """One child per builder session, serving the event loop of the first call that reaches it.""" + + # The child's entry point; tests put a fake child here before `start()`. + _target: Callable[[Connection], None] | None = None + ran_in_child: int + + def __init__( + self, + *, + scan_root: Path | None, + source_root: Path | None, + subfolder: str | None, + mode: Mode | None = None, + stand_in: str | None = None, + ) -> None: + self._scan_root, self._source_root, self._subfolder = scan_root, source_root, subfolder + self._mode: Mode = mode or default_mode() + self._stand_in = stand_in + self.ran_in_child = 0 + # Every child this checker started, so a test can check none outlives it. + self.pids: list[int] = [] + # Guards the cache and the child's state; never held across a send, a join or a wait. + self._cache_lock = threading.Lock() + self._send_lock = threading.Lock() + self._scan: ScanAnswer | None = None + self._instrumentation: str | None = None + self._failures = {part: 0 for part in _PARTS} + self._failure: dict[str, tuple[str, str]] = {} + self._watchers: list[tuple[asyncio.AbstractEventLoop, asyncio.Future[None]]] = [] + self._child: _Child | None = None + self._session: _Session | None = None + self._ids = itertools.count(1) + self._started = False + self._closed = False + self._thread_reason: str | None = None + self._stand_in_error: tuple[str, str] | None = None + self._trial_passed = False + self._deaths = 0 + self._start_thread: threading.Thread | None = None + self._loop: asyncio.AbstractEventLoop | None = None + self._lock: asyncio.Lock | None = None + self._warned: set[str] = set() + # Thread mode's own scan, one shared task per loop, as the contract tool kept it. + self._thread_task: asyncio.Future[tuple[list[str], frozenset[str]]] | None = None + self._reach: list[Path] = [] + self._scanned_here = False + wanted = scan_root is not None or source_root is not None + if ( + wanted + and mode is None + and self._mode == "thread" + and os.environ.get("ALK_SAVE_CHECKS") != "thread" + ): + self._fallback("forkserver is not available on this platform") + + # -- the API --------------------------------------------------------------------------- + + def start(self) -> None: + with self._cache_lock: + if self._started or self._closed or not self._wants_child(): + return + self._started = True + _OPEN.add(self) + # The preload import blocks for up to a second or two, so never on the caller's loop. + self._spawn(self._launch) + + async def scan(self, *, deadline: float) -> ScanAnswer | NotInTime: + loop = asyncio.get_running_loop() + if self._in_thread(loop): + return await self._thread_scan() + self.start() + seen = self._seen() + answer = await self._until(lambda: self._scan_ready(seen), deadline, session=True) + if isinstance(answer, _Thread): + self._note_closed() + return await self._thread_scan() + if answer is None: + logger.warning("save checker: the save's deadline passed in %s; nothing saved", "session") + return NotInTime(_late(_SCAN_LATE)) + return answer + + async def check( + self, + payload: dict[str, Any], + *, + require_setup_contract: bool, + asked: set[str], + held: dict[str, str], + deadline: float, + ) -> Saved | Refused | NotInTime: + loop = asyncio.get_running_loop() + if self._in_thread(loop): + return await self._thread_check(payload, require_setup_contract, asked, held) + self.start() + seen = self._seen() + parts = await self._until(lambda: self._parts_ready(seen), deadline, session=True) + if isinstance(parts, _Thread): + self._note_closed() + return await self._thread_check(payload, require_setup_contract, asked, held) + if parts is None: + logger.warning("save checker: the save's deadline passed in %s; nothing saved", "session") + return NotInTime(_late(_SAVE_LATE)) + lock = self._lock + assert lock is not None + try: + await _by(lock.acquire(), deadline) + except asyncio.TimeoutError: + logger.warning( + "save checker: the save's deadline passed in %s; nothing saved", ", ".join(STAGES) + ) + return NotInTime(_late(_SAVE_LATE)) + if loop.time() >= deadline: + # The lock came with no time left: sending would only get a healthy child killed. + lock.release() + logger.warning( + "save checker: the save's deadline passed in %s; nothing saved", ", ".join(STAGES) + ) + return NotInTime(_late(_SAVE_LATE)) + try: + result = await self._save(payload, require_setup_contract, asked, held, deadline) + finally: + lock.release() + if isinstance(result, _Thread): + self._note_closed() + return await self._thread_check(payload, require_setup_contract, asked, held) + return result + + def close(self) -> None: + """Kills the child and stops its threads. Never raises.""" + child = thread = None + try: + with self._cache_lock: + self._closed = True + child, thread = self._child, self._start_thread + session, self._session = self._session, None + if session is not None and session.timer is not None: + session.timer.cancel() + except Exception: + logger.warning("save checker: closing could not read its state", exc_info=True) + try: + if child is not None: + self._kill(child, "close", restart=False) + except Exception: + logger.warning("save checker: the child could not be killed", exc_info=True) + try: + if thread is not None and thread is not threading.current_thread(): + # A start still running sees the close and kills what it started. + thread.join(JOIN_SECONDS) + except Exception: + logger.warning("save checker: the start thread could not be joined", exc_info=True) + try: + with self._cache_lock: + last = self._child + for one in {c for c in (child, last) if c is not None}: + self._reap(one) + except Exception: + logger.warning("save checker: the child could not be reaped", exc_info=True) + try: + self._notify() + except Exception: + logger.warning("save checker: waiting calls could not be woken", exc_info=True) + + # -- which path a call takes ----------------------------------------------------------- + + def _wants_child(self) -> bool: + return self._mode == "child" and ( + self._scan_root is not None or self._source_root is not None + ) + + def _in_thread(self, loop: asyncio.AbstractEventLoop) -> bool: + if self._closed: + self._note_closed() + return True + if not self._wants_child() or self._thread_reason is not None: + return True + self._raise_stand_in() + if self._loop is None: + self._loop, self._lock = loop, asyncio.Lock() + elif self._loop is not loop: + # The child serves one loop; another loop's call keeps today's in-process path. + self._warn_once( + "other loop", "save checker: a call from a second event loop runs in this process" + ) + return True + return False + + def _note_closed(self) -> None: + if self._closed: + self._warn_once("closed", "save checker: closed; running the checks in this process") + + def _warn_once(self, key: str, text: str) -> None: + with self._cache_lock: + if key in self._warned: + return + self._warned.add(key) + logger.warning(text) + + def _raise_stand_in(self) -> None: + error = self._stand_in_error + if error is not None: + raise StandInFailed(*error) + + # -- thread mode ----------------------------------------------------------------------- + + async def _thread_scan(self) -> ScanAnswer: + with self._cache_lock: + cached = self._scan + if cached is not None: + return cached + from . import tools # tools imports this module + + loop = asyncio.get_running_loop() + scan = self._thread_task + # Run again only a scan that can't give its answer here: one that raised or was + # cancelled, or one on another loop, which this one can't wait on. + if ( + scan is None + or scan.get_loop() is not loop + or (scan.done() and (scan.cancelled() or scan.exception() is not None)) + ): + scan = asyncio.ensure_future( + asyncio.to_thread( + tools._tools_in_agent_folder, self._scan_root, self._subfolder, self._reach + ) + ) + self._thread_task = scan + # Shielded: one call cancelled mustn't cancel the scan the others wait on. + found, cut = await asyncio.shield(scan) + self._scanned_here = True + return ScanAnswer(found, cut) + + async def _thread_check( + self, + payload: dict[str, Any], + require_setup_contract: bool, + asked: set[str], + held: dict[str, str], + ) -> Saved | Refused: + from . import tools # tools imports this module + + with self._cache_lock: + instrumentation = self._instrumentation + extra: dict[str, Any] = {} + if instrumentation is not None: + extra["instrumentation"] = instrumentation + checked = await asyncio.to_thread( + tools._checked_contract, + payload, + source_root=self._source_root, + require_setup_contract=require_setup_contract, + subfolder=self._subfolder, + # Only a reach this process walked is whole; a cached scan's reach stayed in the child. + reached=self._reach + if self._scan_root == self._source_root and self._scanned_here + else None, + asked=asked, + held=held, + **extra, + ) + if isinstance(checked, dict): + return Refused(checked) + return Saved(checked) + + # -- waiting on the loop --------------------------------------------------------------- + + def _seen(self) -> dict[str, int]: + with self._cache_lock: + return dict(self._failures) + + def _has(self, part: str) -> bool: + return (self._scan if part == "scan" else self._instrumentation) is not None + + def _gave_way(self) -> bool: + return self._closed or self._thread_reason is not None + + def _raise_failed(self, part: str, seen: dict[str, int]) -> None: + if not self._has(part) and self._failures[part] > seen[part]: + raise SessionChecksFailed(part, *self._failure[part]) + + def _scan_ready(self, seen: dict[str, int]) -> ScanAnswer | _Thread | None: + if self._gave_way(): + return _THREAD + self._raise_stand_in() + if self._scan is not None: + return self._scan + self._raise_failed("scan", seen) + return None + + def _parts_ready(self, seen: dict[str, int]) -> bool | _Thread | None: + if self._gave_way(): + return _THREAD + self._raise_stand_in() + if self._scan is not None and self._instrumentation is not None: + return True + for part in _PARTS: + self._raise_failed(part, seen) + return None + + def _child_ready(self) -> _Child | _Thread | None: + if self._gave_way(): + return _THREAD + self._raise_stand_in() + child = self._child + if ( + child is not None + and child.ready + and not child.gone + and child.killed is None + and child.request is None + ): + return child + return None + + async def _until(self, condition: Callable[[], Any], deadline: float, *, session: bool = False) -> Any: + """``condition()``'s first answer that isn't None, or None once ``deadline`` passes.""" + loop = asyncio.get_running_loop() + while True: + if session: + self._ask_session() + future: asyncio.Future[None] = loop.create_future() + entry = (loop, future) + with self._cache_lock: + # Registered and read under one lock, so an answer can't land between the two. + answer = condition() + if answer is None: + self._watchers.append(entry) + if answer is not None: + return answer + try: + await _by(future, deadline) + except asyncio.TimeoutError: + with self._cache_lock: + return condition() + finally: + with self._cache_lock: + if entry in self._watchers: + self._watchers.remove(entry) + + def _notify(self) -> None: + with self._cache_lock: + watchers, self._watchers = self._watchers, [] + for loop, future in watchers: + try: + loop.call_soon_threadsafe(_settle, future) + except RuntimeError: + # Its loop has closed; the cache is already written for a later call. + pass + + # -- a save in the child --------------------------------------------------------------- + + async def _save( + self, + payload: dict[str, Any], + require_setup_contract: bool, + asked: set[str], + held: dict[str, str], + deadline: float, + ) -> Saved | Refused | NotInTime | _Thread: + loop = asyncio.get_running_loop() + inbox: asyncio.Queue[dict[str, Any]] = asyncio.Queue() + + def reply(message: dict[str, Any]) -> None: + try: + loop.call_soon_threadsafe(inbox.put_nowait, message) + except RuntimeError: + pass + + while True: + child = await self._until(self._child_ready, deadline) + if isinstance(child, _Thread): + return child + if child is None or loop.time() >= deadline: + # A child that came up in the deadline's tick isn't sent the save: it would only + # be cut, and a healthy child killed for it. + logger.warning( + "save checker: the save's deadline passed in %s; nothing saved", + ", ".join(STAGES), + ) + return NotInTime(_late(_SAVE_LATE)) + rid = next(self._ids) + with self._cache_lock: + if child.gone or child.killed is not None: + continue + child.replies[rid] = reply + child.request = rid + instrumentation = self._instrumentation + break + request = { + "id": rid, + "kind": "save", + "payload": payload, + "source_root": _text(self._source_root), + "subfolder": self._subfolder, + "require_setup_contract": require_setup_contract, + "instrumentation": instrumentation, + "asked": sorted(asked), + "held": dict(held), + # A restarted child gets no `session`, so each save carries the level too. + "log_level": logging.getLogger().level, + } + if self._send(child, request): + self.ran_in_child += 1 + problems: list[str] = [] + answered: list[str] = [] + last: dict[str, Any] | None = None + outcome: tuple[str, dict[str, Any] | None] | None = None + + def read(message: dict[str, Any]) -> tuple[str, dict[str, Any] | None] | None: + nonlocal last + kind = message.get("kind") + if kind == "gone": + return "gone", None + if message.get("id") != rid: + return None + if kind == "stage": + answered.append(str(message.get("stage"))) + problems.extend(message.get("problems") or []) + last = message + return None + if kind in ("done", "failed"): + return str(kind), message + return None + + try: + while outcome is None: + try: + message = await _by(inbox.get(), deadline) + except asyncio.TimeoutError: + # Answers the reader handed over in this tick still count: `done` wins a tie. + await asyncio.sleep(0) + while outcome is None and not inbox.empty(): + outcome = read(inbox.get_nowait()) + if outcome is None: + outcome = ("cut", None) + break + outcome = read(message) + except asyncio.CancelledError: + if outcome is None: + # Only this call's own request is killed; it isn't a death. + self._kill(child, "cancel", restart=True) + raise + finally: + with self._cache_lock: + child.replies.pop(rid, None) + if child.request == rid: + child.request = None + kind, message = outcome + if kind == "done": + assert message is not None + with self._cache_lock: + self._deaths = 0 + asked.update(message.get("asked") or []) + held.update(message.get("held") or {}) + if message.get("contract") is not None: + return Saved(AgentContract.model_validate(message["contract"])) + return Refused(message["response"]) + if kind == "failed": + assert message is not None + with self._cache_lock: + self._deaths = 0 + _log_failed("save", message) + raise SaveChecksFailed(str(message.get("error")), str(message.get("message"))) + if kind == "cut": + logger.warning( + "save checker: the save's deadline passed in %s; nothing saved", + ", ".join(stage for stage in STAGES if stage not in answered), + ) + self._kill(child, "deadline", restart=True) + # Cut, or the child died: the model sees only the problems that arrived. + if problems and last is not None: + from .tools import _problems # tools imports this module + + asked.update(last.get("asked") or []) + held.update(last.get("held") or {}) + arrived = sorted(payload) if isinstance(payload, dict) else [type(payload).__name__] + return Refused(_problems(problems, arrived)) + return NotInTime(_late(_SAVE_LATE)) + + # -- the session job ------------------------------------------------------------------- + + def _ask_session(self) -> None: + """Sends `session` when a part is missing, the child is ready and none is in flight.""" + with self._cache_lock: + child = self._child + if ( + self._session is not None + or (self._scan is not None and self._instrumentation is not None) + or child is None + or not child.ready + or child.gone + or child.killed is not None + or child.request is not None + or self._gave_way() + or self._stand_in_error is not None + ): + return + session = _Session(next(self._ids), child, set(_PARTS)) + session.timer = threading.Timer(SESSION_BOUND, self._bound, args=(session,)) + session.timer.daemon = True + self._session = session + child.request = session.id + session.timer.start() + self._send( + child, + { + "id": session.id, + "kind": "session", + "scan_root": _text(self._scan_root), + "source_root": _text(self._source_root), + "subfolder": self._subfolder, + "log_level": logging.getLogger().level, + }, + ) + + def _cut(self, part: str) -> None: + """Caches ``part`` as cut, unless it answered: the first write wins. Under the lock.""" + if part == "scan" and self._scan is None: + self._scan = ScanAnswer([], frozenset({"deadline"})) + elif part == "instrumentation" and self._instrumentation is None: + self._instrumentation = "none" + + def _end_session(self, session: _Session) -> list[str]: + """Takes ``session`` out of flight and caches its missing parts as cut. Under the lock.""" + self._session = None + if session.timer is not None: + session.timer.cancel() + if session.child.request == session.id: + session.child.request = None + missing = [part for part in _PARTS if not self._has(part)] + for part in missing: + self._cut(part) + return missing + + def _bound(self, session: _Session) -> None: + child = session.child + with self._cache_lock: + if self._session is not session: + return + missing = self._end_session(session) + # Marked killed in the same hold that ends the session, so no call, waiting or new, + # sends a save to a child about to go. + ending = child.killed is None and not child.gone + if ending: + child.killed = "bound" + if missing: + logger.warning( + "save checker: the session's checks ran past %d s; %s counts as cut", + SESSION_BOUND, + _which(missing), + ) + if ending: + try: + child.process.kill() + except Exception: + logger.warning("save checker: the child could not be killed", exc_info=True) + self._spawn(self._replace, child) + self._notify() + + def _part(self, child: _Child, message: dict[str, Any]) -> None: + kind = message.get("kind") + part = str(message.get("part")) if kind == "failed" else str(kind) + cut_now = False + with self._cache_lock: + session = self._session + if session is None or session.child is not child or message.get("id") != session.id: + return + session.pending.discard(part) + if kind == "failed": + if not self._has(part): + self._failures[part] += 1 + if self._failures[part] >= 2: + self._cut(part) + cut_now = True + else: + self._failure[part] = ( + str(message.get("error")), + str(message.get("message")), + ) + elif part == "scan" and self._scan is None: + self._scan = ScanAnswer( + list(message.get("found") or []), frozenset(message.get("cut") or []) + ) + elif part == "instrumentation" and self._instrumentation is None: + self._instrumentation = str(message.get("value")) + if not session.pending: + self._session = None + if session.timer is not None: + session.timer.cancel() + if child.request == session.id: + child.request = None + if kind == "failed": + _log_failed(part, message) + if cut_now: + logger.warning("save checker: the session's %s failed twice; it counts as cut", part) + self._notify() + + # -- the child's life ------------------------------------------------------------------ + + def _spawn(self, work: Callable[..., None], *args: Any) -> None: + thread = threading.Thread( + target=self._guarded, args=(work, *args), name="save-checker-start", daemon=True + ) + with self._cache_lock: + self._start_thread = thread + thread.start() + + def _guarded(self, work: Callable[..., None], *args: Any) -> None: + try: + work(*args) + except Exception: + logger.error("save checker: the start thread failed", exc_info=True) + + def _launch(self) -> None: + try: + child = self._new_child() + except Exception as error: + logger.debug("save checker: the child could not start", exc_info=True) + self._fallback(f"the child could not start ({type(error).__name__})") + return + if child is not None: + self._bring_up(child) + + def _new_child(self) -> _Child | None: + target = self._target + if target is None: + from . import save_child # it imports tools, which imports this module + + target = save_child.serve + context = multiprocessing.get_context("forkserver") + # The fork server is one per process and the first caller fixes its preload, so both + # checkers that use it name the same modules. + context.set_forkserver_preload(PRELOAD) + parent, child_end = context.Pipe() + try: + process = context.Process(target=target, args=(child_end,), daemon=True) + process.start() + except BaseException: + parent.close() + raise + finally: + # The child has its own copy. Closing ours lets a dead child read as end of file. + child_end.close() + assert process.pid is not None + child = _Child(process, parent, process.pid) + with self._cache_lock: + self.pids.append(process.pid) + closed = self._closed + if not closed: + self._child = child + if closed: + # A close that landed while this started: it found nothing to kill, so this does. + child.killed = "close" + self._reap(child) + return None + child.reader = threading.Thread( + target=self._read, args=(child,), name="save-checker-reader", daemon=True + ) + child.reader.start() + threading.Thread( + target=self._watch, args=(child,), name="save-checker-watchdog", daemon=True + ).start() + return child + + def _bring_up(self, child: _Child) -> None: + up = self._next(child, time.monotonic() + START_SECONDS) + if up.get("kind") != "up": + # One that is still alive is ended first. After a passed trial it counts as a death, + # as the watchdog's kills do, so a child that never comes up can't hold the session. + self._kill(child, "start" if self._trial_passed else "trial", restart=False) + self._lost(child, f"the child wasn't up within {START_SECONDS:g} s, or ended first") + return + since = time.monotonic() + if not self._trial_passed: + answer = self._ask(child, {"kind": "trial"}, since + TRIAL_SECONDS) + if answer is None: + self._lost(child, f"the trial did not answer within {TRIAL_SECONDS:g} s") + return + if answer.get("kind") != "trial": + self._lost(child, "the child ended before the trial answered") + return + if not answer.get("ok"): + self._lost(child, f"the trial failed: {answer.get('detail')}") + return + self._trial_passed = True + if self._stand_in: + answer = self._ask( + child, + {"kind": "stand_in", "module": self._stand_in}, + time.monotonic() + TRIAL_SECONDS, + ) + if answer is not None and answer.get("kind") == "gone": + self._lost(child, "the child ended before the stand-in answered") + return + if answer is None or not answer.get("ok"): + detail = "no answer in time" if answer is None else str(answer.get("detail")) + with self._cache_lock: + self._stand_in_error = (self._stand_in, detail) + self._kill(child, "stand_in", restart=False) + self._reap(child) + self._notify() + return + with self._cache_lock: + gone = child.gone + if not gone: + child.ready = True + if gone: + self._ended(child) + return + self._notify() + self._ask_session() + + def _lost(self, child: _Child, reason: str) -> None: + """The child stopped before it was ready: a failed trial, or a death after one passed.""" + if child.killed == "close": + return + if self._trial_passed: + self._ended(child) + return + self._fallback(reason) + self._kill(child, "trial", restart=False) + self._reap(child) + + def _next(self, child: _Child, until: float | None = None) -> dict[str, Any]: + """The child's next message to the start thread; `gone` when it ends or is closed, or + when ``until`` passes first.""" + while True: + try: + return child.inbox.get(timeout=0.1) + except queue.Empty: + if self._closed or (until is not None and time.monotonic() > until): + return _GONE + + def _ask(self, child: _Child, request: dict[str, Any], until: float) -> dict[str, Any] | None: + rid = next(self._ids) + with self._cache_lock: + if child.gone or child.killed is not None: + return _GONE + child.replies[rid] = child.inbox.put + child.request = rid + answer: dict[str, Any] | None = None + try: + self._send(child, {"id": rid, **request}) + while True: + try: + message = child.inbox.get(timeout=max(until - time.monotonic(), 0.0)) + except queue.Empty: + break + if message.get("kind") == "gone" or message.get("id") == rid: + answer = message + break + finally: + with self._cache_lock: + child.replies.pop(rid, None) + if child.request == rid: + child.request = None + return answer + + def _send(self, child: _Child, message: dict[str, Any]) -> bool: + try: + with self._send_lock: + child.conn.send(message) + return True + except (OSError, ValueError): + # A child that ended; its reader reports that. + return False + + def _read(self, child: _Child) -> None: + while True: + try: + message = child.conn.recv() + except Exception: + # End of file, a closed pipe, or a message that won't unpickle: the child is gone. + break + try: + self._dispatch(child, message) + except Exception: + logger.error("save checker: a message from the child could not be handled", exc_info=True) + self._gone(child) + + def _dispatch(self, child: _Child, message: Any) -> None: + if not isinstance(message, dict): + return + kind = message.get("kind") + if kind == "log": + self._relog(message) + return + if kind == "up": + child.inbox.put(message) + return + if kind in _PARTS or (kind == "failed" and message.get("part") in _PARTS): + self._part(child, message) + return + with self._cache_lock: + reply = child.replies.get(message.get("id")) + if reply is not None: + reply(message) + + def _relog(self, message: dict[str, Any]) -> None: + name = str(message.get("name") or __name__) + level = message.get("level") + level = level if isinstance(level, int) else logging.WARNING + target = logging.getLogger(name) + if not target.isEnabledFor(level): + return + text = str(message.get("message", "")) + if message.get("trace"): + text += "\n" + str(message["trace"]) + record = target.makeRecord(name, level, "(save checker child)", 0, text, None, None) + # Masked here too: the child's logger may be one this process never set a filter on. + MaskSecrets().filter(record) + target.handle(record) + + def _gone(self, child: _Child) -> None: + with self._cache_lock: + child.gone = True + ready = child.ready + replies = list(child.replies.values()) + child.replies.clear() + child.stop.set() + child.inbox.put(_GONE) + for reply in replies: + reply(_GONE) + # Before it was ready, the start thread decides what its end means. + if ready: + self._ended(child) + + def _ended(self, child: _Child) -> None: + """A ready child ended. Unless the parent killed it, that is a death.""" + with self._cache_lock: + if child.handled: + return + child.handled = True + reason = child.killed + if reason not in (None, "watchdog", "start"): + return + self._deaths += 1 + deaths = self._deaths + missing: list[str] = [] + session = self._session + if session is not None and session.child is child: + missing = self._end_session(session) + code = reason if reason in ("watchdog", "start") else self._exit_code(child) + logger.error("save checker: the child ended (%s)", code) + if missing: + logger.warning( + "save checker: the child ended during the session's checks; %s counts as cut", + _which(missing), + ) + self._notify() + if deaths >= DEATHS: + self._fallback(f"the child ended {deaths} times in a row") + self._spawn(self._reap, child) + else: + self._spawn(self._replace, child) + + def _exit_code(self, child: _Child) -> str: + try: + child.process.join(0.5) + code = child.process.exitcode + except (OSError, ValueError, AssertionError): + return "exit code unknown" + return "still running" if code is None else f"exit code {code}" + + def _kill(self, child: _Child, reason: str, *, restart: bool) -> bool: + """Kills ``child`` for ``reason``; False when it had already ended or been killed.""" + with self._cache_lock: + if child.killed is not None or child.gone: + return False + child.killed = reason + try: + child.process.kill() + except Exception: + logger.warning("save checker: the child could not be killed", exc_info=True) + if restart: + self._spawn(self._replace, child) + return True + + def _replace(self, old: _Child) -> None: + # The old reader ends before the new one starts, so one reader reads at a time. + self._reap(old) + with self._cache_lock: + stop = self._gave_way() or self._stand_in_error is not None + if not stop: + self._launch() + + def _reap(self, child: _Child) -> None: + """Kills, joins and closes ``child``, its pipe and its reader. Never raises.""" + with self._cache_lock: + if child.reaped: + return + child.reaped = True + if child.killed is None and not child.gone: + child.killed = "reap" + child.stop.set() + process = child.process + try: + process.kill() + process.join(JOIN_SECONDS) + except Exception: + logger.warning("save checker: the child could not be ended", exc_info=True) + reader = child.reader + if reader is not None and reader is not threading.current_thread(): + reader.join(JOIN_SECONDS) + try: + child.conn.close() + except OSError: + logger.debug("save checker: the pipe was already closed", exc_info=True) + try: + # Raises when the join above timed out; the child is then left to the exit handler. + process.close() + except Exception: + logger.warning("save checker: the child could not be closed", exc_info=True) + with self._cache_lock: + if self._child is child: + self._child = None + + def _fallback(self, reason: str) -> None: + with self._cache_lock: + if self._thread_reason is not None: + return + self._thread_reason = reason + logger.warning("save checker: running the checks in this process: %s", reason) + self._notify() + + def _watch(self, child: _Child) -> None: + while not child.stop.wait(WATCH_INTERVAL): + with self._cache_lock: + busy = child.request is not None and child.killed is None and not child.gone + if not busy: + continue + rss = _rss(child.pid) + if rss is not None and rss > WATCH_CAP: + self._kill(child, "watchdog", restart=False) + return diff --git a/src/fi/alk/harness/save_child.py b/src/fi/alk/harness/save_child.py new file mode 100644 index 000000000..eceae8b74 --- /dev/null +++ b/src/fi/alk/harness/save_child.py @@ -0,0 +1,424 @@ +"""Runs the contract tool's checks in a child process the save checker can stop. + +The checks read the customer's files, and Python's parser holds the GIL while it reads one, so +in a thread of the conversation's process a crafted repository can hold the conversation for +seconds. In a child, the parent stops waiting at the save's deadline and kills it. The child runs +the same ``_checked_contract`` the thread fallback runs, so both paths share one code path. + +Every check is looked up through its module when it is called, so a test's stand-in, which +replaces a module's attribute in the child, reaches it. +""" + +from __future__ import annotations + +import copy +import importlib +import logging +import os +import shutil +import tempfile +import threading +import traceback +from collections.abc import Callable +from multiprocessing.connection import Connection +from pathlib import Path +from typing import Any + +from . import agent_type_check, instrumentation_evidence, source_folder, tools +from .contract import AgentContract +from .reading_entry import MaskSecrets + +logger = logging.getLogger(__name__) +logger.addFilter(MaskSecrets()) + +# The trial's repository: the folder imports a browser tool, and the traced import sits in code +# outside the folder, so the trial passes only when the scan finds something and instrumentation +# reads what the scan reached. A tree with neither passes even when the child's scan is broken. +TRIAL_TREE = { + "agent/main.py": "from lib import trace\nimport playwright\n\n\ndef run(x):\n return x\n", + "agent/compose.yaml": "services:\n agent:\n build: .\n", + "lib/__init__.py": "", + "lib/trace.py": "import traceai_openai\n", +} +TRIAL_SUBFOLDER = "agent" +TRIAL_PAYLOAD: dict[str, Any] = { + "agent": "trial", + "modality": "chat", + "real_use_cases": ["o"], + "hard_constraints": ["be polite"], + "tools": [{"name": "t0", "args": ["x"]}], + "tool_entrypoints": [{"tool": "t0", "mode": "import", "module": "main", "callable": "run"}], + "system_prompt_from": [{"path": "agent/main.py", "start_line": 1, "end_line": 3}], + "data_store": {"kind": "none"}, + "runtime": {"language": "python", "compose_file": "agent/compose.yaml"}, +} +# The order _checked_contract reports its stages in; the trial holds the child to it. +_STAGE_ORDER = ["contract", "tool_evidence", "prompt_copy", "model", "runtime"] +# A stand-in may only replace the kit's own checks. +_PACKAGE = "fi.alk.harness" + +Send = Callable[[dict[str, Any]], None] + + +class _PipeGone(BaseException): + """The parent's end of the pipe is closed. A BaseException, so that no check's own + ``except Exception`` mistakes it for a fault in the customer's files: the serve loop ends.""" + + +class _Pipe: + """The child's end of the pipe, shared by the serve loop and the log handler. One lock + around each send, so a line logged from another thread never lands inside a reply.""" + + def __init__(self, conn: Connection) -> None: + self.conn = conn + self._lock = threading.RLock() + + def send(self, message: dict[str, Any]) -> None: + try: + with self._lock: + self.conn.send(message) + except OSError as gone: + raise _PipeGone from gone + + +class PipeLogHandler(logging.Handler): + """Sends each record to the parent as text, which logs it on its own logger of that name. + A record, an exception or a path can't cross the pipe, so only strings and numbers do.""" + + def __init__(self, pipe: _Pipe) -> None: + super().__init__() + # No MaskSecrets here: reading the secret values can log, and on the root that line would + # come back through this handler's mask again. The kit's loggers mask their own lines. + self._pipe = pipe + + def emit(self, record: logging.LogRecord) -> None: + try: + if record.exc_text: + # A masked logger keeps a trace only as this text, having cleared exc_info. + trace: str | None = record.exc_text + elif record.exc_info: + trace = logging.Formatter().formatException(record.exc_info) + else: + trace = None + message = { + "id": None, + "kind": "log", + "name": record.name, + "level": record.levelno, + "message": record.getMessage(), + "trace": trace, + } + except Exception: + self.handleError(record) + return + try: + self._pipe.send(message) + except _PipeGone: + # The parent closed its end, so this child is going away; its last lines go with it. + pass + + +class _Held(dict[str, str]): + """The reading's held quotes, recording every key the save writes. Only those go back, so + a quote another call wrote meanwhile isn't overwritten with the one this call was given.""" + + def __init__(self, given: dict[str, str]) -> None: + super().__init__(given) + self.written: set[str] = set() + + def __setitem__(self, key: str, value: str) -> None: + super().__setitem__(key, value) + self.written.add(key) + + def update(self, *args: Any, **kwargs: Any) -> None: # type: ignore[override] + for key, value in dict(*args, **kwargs).items(): + self[key] = value + + def setdefault(self, key: str, default: Any = None) -> Any: # type: ignore[override] + if key not in self: + self[key] = default + return self[key] + + def delta(self) -> dict[str, str]: + return {key: self[key] for key in sorted(self.written) if key in self} + + +def _path(value: str | None) -> Path | None: + return Path(value) if value is not None else None + + +def _failed(rid: int, part: str, error: BaseException) -> dict[str, Any]: + return { + "id": rid, + "kind": "failed", + "part": part, + "error": type(error).__name__, + "message": str(error), + "trace": traceback.format_exc(), + } + + +def _session_job( + send: Send, + rid: int, + scan_root: Path | None, + source_root: Path | None, + subfolder: str | None, +) -> None: + """The browser scan and the instrumentation check, each sent as soon as it answers.""" + reach: list[Path] = [] + scanned = False + + def scan() -> None: + nonlocal scanned + found, cut = tools._tools_in_agent_folder(scan_root, subfolder, reach) + scanned = True + send({"id": rid, "kind": "scan", "found": list(found), "cut": sorted(cut)}) + + def instrumentation() -> None: + send({"id": rid, "kind": "instrumentation", "value": traced()}) + + def traced() -> str: + if source_root is None: + return "none" + if not subfolder: + return instrumentation_evidence.detect_instrumentation(source_root) + try: + folder = source_folder.agent_folder(source_root, subfolder) + # A scan that raised may have filled only part of its reach, and one of another + # root reached other files: either way the agent's reach is walked afresh. + reached = ( + reach + if scanned and scan_root is not None and scan_root == source_root + else agent_type_check.code_the_agent_reaches(source_root, subfolder) + ) + except source_folder.SubfolderNotFound: + # The save refuses the folder itself; instrumentation has nothing to read. + return "none" + return instrumentation_evidence.detect_instrumentation(folder, reached) + + # With a folder, instrumentation reads what the scan reached, so the scan goes first. + parts = ( + [("scan", scan), ("instrumentation", instrumentation)] + if subfolder + else [("instrumentation", instrumentation), ("scan", scan)] + ) + for part, run in parts: + try: + run() + except Exception as error: + send(_failed(rid, part, error)) + + +def _save_job(send: Send, request: dict[str, Any]) -> None: + """``_checked_contract`` on one payload: a ``stage`` per stage as it ends, then one + ``done``, or ``failed`` when it raised.""" + rid = request["id"] + try: + asked = set(request["asked"]) + held = _Held(request["held"]) + + def report(stage: str, problems: list[str]) -> None: + send( + { + "id": rid, + "kind": "stage", + "stage": stage, + "problems": [str(problem) for problem in problems], + "asked": sorted(asked), + "held": held.delta(), + } + ) + + result = tools._checked_contract( + request["payload"], + source_root=_path(request["source_root"]), + require_setup_contract=bool(request["require_setup_contract"]), + subfolder=request["subfolder"], + instrumentation=request["instrumentation"], + asked=asked, + held=held, + progress=report, + ) + if isinstance(result, AgentContract): + contract, response = result.model_dump(mode="json"), None + elif isinstance(result, dict): + contract, response = None, result + else: + raise TypeError( + f"the save checks answered with a {type(result).__name__}, " + "not a contract or a response" + ) + except Exception as error: + send(_failed(rid, "save", error)) + return + send( + { + "id": rid, + "kind": "done", + "contract": contract, + "response": response, + "asked": sorted(asked), + "held": held.delta(), + } + ) + + +def _trial_mismatch() -> str: + """Runs both jobs on the trial's tree, keeping their messages here, and names the first + thing that isn't as the real checks answer it; empty when everything is.""" + heard: list[dict[str, Any]] = [] + root = Path(tempfile.mkdtemp(prefix="alk-trial-")) + try: + for name, text in TRIAL_TREE.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + _session_job(heard.append, 0, root, root, TRIAL_SUBFOLDER) + value = next((m["value"] for m in heard if m["kind"] == "instrumentation"), "none") + _save_job( + heard.append, + { + "id": 0, + "payload": copy.deepcopy(TRIAL_PAYLOAD), + "source_root": str(root), + "subfolder": TRIAL_SUBFOLDER, + # The trial's payload declares no setup, which the stricter check refuses. + "require_setup_contract": False, + "instrumentation": value, + "asked": [], + "held": {}, + }, + ) + finally: + shutil.rmtree(root, ignore_errors=True) + return _first_mismatch(heard) + + +def _answer_of(heard: list[dict[str, Any]], kind: str, part: str) -> tuple[dict[str, Any], str]: + for message in heard: + if message["kind"] == kind: + return message, "" + if message["kind"] == "failed" and message["part"] == part: + return message, f"the {part} raised {message['error']}: {message['message']}" + return {}, f"no {kind} answer" + + +def _first_mismatch(heard: list[dict[str, Any]]) -> str: + scan, why = _answer_of(heard, "scan", "scan") + if why: + return why + if scan["found"] != ["playwright"]: + return f"the scan found {scan['found']}, not ['playwright']" + traced, why = _answer_of(heard, "instrumentation", "instrumentation") + if why: + return why + if traced["value"] != "traceai": + return f"instrumentation read {traced['value']!r}, not 'traceai'" + done, why = _answer_of(heard, "done", "save") + if why: + return why + stages = [m["stage"] for m in heard if m["kind"] == "stage"] + if stages != _STAGE_ORDER: + return f"the save reported the stages {stages}, not {_STAGE_ORDER}" + if heard[-1] is not done: + return "the save answered before its last stage" + if done["contract"] is None or done["response"] is not None: + return "the save refused the trial's contract" + return "" + + +def _trial(send: Send, request: dict[str, Any]) -> None: + try: + detail = _trial_mismatch() + except Exception as error: + # The trial's own fault fails the trial; it isn't a part of a job to report as failed. + detail = f"{type(error).__name__}: {error}" + send({"id": request["id"], "kind": "trial", "ok": not detail, "detail": detail}) + + +def _install_stand_in(name: str) -> None: + """Replaces each ``(module, attribute)`` in the stand-in's ``STAGES`` with its value, all or + none, so a stand-in naming a check that isn't there changes nothing.""" + stages = importlib.import_module(name).STAGES + found = [] + for (target, attribute), value in stages.items(): + full = target if target.startswith(f"{_PACKAGE}.") else f"{_PACKAGE}.{target}" + module = importlib.import_module(full) + if not hasattr(module, attribute): + raise AttributeError(f"{full} has no {attribute} to stand in for") + found.append((module, attribute, value)) + for module, attribute, value in found: + setattr(module, attribute, value) + + +def _stand_in(send: Send, request: dict[str, Any]) -> None: + try: + _install_stand_in(request["module"]) + except Exception as error: + detail = f"{type(error).__name__}: {error}" + send({"id": request["id"], "kind": "stand_in", "ok": False, "detail": detail}) + return + send({"id": request["id"], "kind": "stand_in", "ok": True, "detail": ""}) + + +def _session(send: Send, request: dict[str, Any]) -> None: + _session_job( + send, + request["id"], + _path(request["scan_root"]), + _path(request["source_root"]), + request["subfolder"], + ) + + +_JOBS: dict[str, Callable[[Send, dict[str, Any]], None]] = { + "trial": _trial, + "stand_in": _stand_in, + "session": _session, + "save": _save_job, +} + + +def serve(conn: Connection, *, install_logging: bool = True) -> None: + """The child's loop: ``up``, then one request at a time, in order, until the parent's end + closes. With ``install_logging`` the child's log lines go to the parent instead of its own + handlers, which are put back, with the level, when the loop ends.""" + pipe = _Pipe(conn) + root = logging.getLogger() + kept_handlers, kept_level = list(root.handlers), root.level + handler = PipeLogHandler(pipe) if install_logging else None + if handler is not None: + for old in kept_handlers: + root.removeHandler(old) + root.addHandler(handler) + try: + pipe.send({"id": None, "kind": "up", "pid": os.getpid()}) + while True: + try: + request = conn.recv() + except (EOFError, OSError): + return + job = _JOBS.get(request.get("kind")) if isinstance(request, dict) else None + if job is None: + # The parent sends only the kinds above and waits on no other, so a reply would + # have no reader. + logger.warning("save child: a request of a kind it doesn't know was dropped") + continue + level = request.get("log_level") + if handler is not None and isinstance(level, int): + # The parent's level, so the child sends only what the parent would log. + root.setLevel(level) + try: + job(pipe.send, request) + except Exception: + # A malformed request; the loop goes on for the next one. + logger.exception("save child: a %s request could not be answered", request["kind"]) + except _PipeGone: + return + finally: + if handler is not None: + root.removeHandler(handler) + for old in kept_handlers: + root.addHandler(old) + root.setLevel(kept_level) diff --git a/src/fi/alk/harness/scan_folders.py b/src/fi/alk/harness/scan_folders.py index 06d7193b4..3b7d4f276 100644 --- a/src/fi/alk/harness/scan_folders.py +++ b/src/fi/alk/harness/scan_folders.py @@ -19,6 +19,7 @@ from pathlib import Path from typing import Any +from .forkserver_preload import PRELOAD from .framework_detection import detect_framework from .reading_entry import MaskSecrets from .source_folder import SubfolderNotFound, agent_folder @@ -27,7 +28,7 @@ logger.addFilter(MaskSecrets()) # Loaded once in the fork server, so a restarted child doesn't import the kit again. -_PRELOAD = ["fi.alk.harness.framework_detection"] +_PRELOAD = PRELOAD # A killed child is gone almost at once; the bound only stops a wedged reap holding the scan. _JOIN_SECONDS = 5.0 # The key of a child's answer that carries a failed check's trace instead of a framework. diff --git a/src/fi/alk/harness/tools.py b/src/fi/alk/harness/tools.py index b467fe55a..fa6b79af1 100644 --- a/src/fi/alk/harness/tools.py +++ b/src/fi/alk/harness/tools.py @@ -11,8 +11,10 @@ import json import logging import secrets +import weakref +from collections.abc import Callable from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any from fi.simulate.agent.wrappers.http_events import ( CompletedToolEvents, @@ -23,8 +25,7 @@ SECOND_LOOK_READ, SecondLook, agent_type_failure, - browser_tools_found, - browser_tools_in_agent, + browser_scan, code_the_agent_reaches, failure_message, is_browser, @@ -73,6 +74,9 @@ from .python_source import utf8_size from .source_folder import SubfolderNotFound, agent_folder, bind_runtime +if TYPE_CHECKING: + from .save_checker import Mode + _LOG = logging.getLogger(__name__) CONTRACT_SERVER = "contract" @@ -209,16 +213,24 @@ def _checked_contract( reached: list[Path] | None = None, asked: set[str] | None = None, held: dict[str, str] | None = None, + instrumentation: str | None = None, + progress: Callable[[str, list[str]], None] | None = None, ) -> AgentContract | dict[str, Any]: """The contract to save, or the problems to send back. It reads the customer's files and writes nothing, so the tool runs it off the event loop. ``reached`` is the code outside the - folder the browser scan found the agent reaches, when it has run.""" + folder the browser scan found the agent reaches, when it has run. ``instrumentation``, when + given, is the answer already found for this session, so it isn't read again. ``progress`` is + told each stage's new problems as the stage ends, so a save cut partway can still answer + with what was found.""" arrived = sorted(payload) if isinstance(payload, dict) else [type(payload).__name__] - payload = _without_nulls(unwrapped(payload)) + payload = _contract_payload(payload) try: contract = AgentContract.model_validate(payload) except Exception as invalid: - return _problems([f"schema:{invalid}"[:600]], arrived) + schema_problems = [f"schema:{invalid}"[:600]] + if progress is not None: + progress("contract", list(schema_problems)) + return _problems(schema_problems, arrived) # Only the save records lines it couldn't confirm. contract.unconfirmed_prompt_from = [] @@ -237,12 +249,38 @@ def _checked_contract( if require_setup_contract and contract.modality == "chat": if problem := generated_chat_interface_problem(contract.runtime): problems.append(problem) + if progress is not None: + progress("contract", list(problems)) with path_lookup_budget(): return _checked_against_source( - contract, problems, arrived, source_root, subfolder, reached, asked, held + contract, + problems, + arrived, + source_root, + subfolder, + reached, + asked, + held, + instrumentation=instrumentation, + progress=progress, ) +def _contract_payload(payload: dict[str, Any]) -> dict[str, Any]: + """The payload as the contract is read from it, shared by the quick check and the full one so + neither refuses what the other accepts.""" + return _without_nulls(unwrapped(payload)) + + +def _quick_schema(payload: dict[str, Any], arrived: list[str]) -> dict[str, Any] | None: + """The schema refusal, answered without waiting on the checks that read files.""" + try: + AgentContract.model_validate(_contract_payload(payload)) + except Exception as invalid: + return _problems([f"schema:{invalid}"[:600]], arrived) + return None + + def _checked_against_source( contract: AgentContract, problems: list[str], @@ -252,6 +290,9 @@ def _checked_against_source( reached: list[Path] | None = None, asked: set[str] | None = None, held: dict[str, str] | None = None, + *, + instrumentation: str | None = None, + progress: Callable[[str, list[str]], None] | None = None, ) -> AgentContract | dict[str, Any]: """The checks that read the customer's files, under the save's one path lookup budget. ``asked`` is the reading's record of what it has been asked once. A model whose citation was @@ -259,6 +300,16 @@ def _checked_against_source( for the quote, rather than refused again: both are optional, and neither may cost the reading its turns. ``held`` is the reading's latest quote (submit_contract keeps it), so lines named later without one are still checked against what was quoted.""" + # Where the stage now running began in problems, so each report holds only its own. + stage, since = "tool_evidence", len(problems) + + def stage_ended(name: str, then: str | None) -> None: + nonlocal stage, since + if progress is not None: + progress(name, problems[since:]) + if then is not None: + stage, since = then, len(problems) + if source_root is not None: from .source_tool_evidence import tool_evidence_problems @@ -280,6 +331,7 @@ def _checked_against_source( set_aside: list[str] = [] try: problems.extend(tool_evidence_problems(contract, source_root, subfolder)) + stage_ended("tool_evidence", "prompt_copy") copy_problems = _copy_prompt(contract, source_root) problems.extend(copy_problems) if contract.system_prompt_from and not copy_problems: @@ -358,6 +410,7 @@ def _checked_against_source( "unconfirmed and keeps the quote, or saves no prompt if the quote has " "fewer than 16 letters" ) + stage_ended("prompt_copy", "model") if cited := model_problems(contract, source_root): if asked is not None and "model" in asked: contract.model = None @@ -366,13 +419,22 @@ def _checked_against_source( if asked is not None: asked.add("model") problems.append("model: if the model is refused again, it is left out") + stage_ended("model", None) except PathLookupsSpent as spent: + if progress is not None: + progress(stage, [*problems[since:], str(spent)]) return _problems([*problems, str(spent)], arrived) + # After the model's report: a save cut later answers without the notes on lines set aside. if problems: problems.extend(set_aside) else: # With no source there is no line to find the name on, so the model is unknown. contract.model = None + # Every stage is reported on every save, so the one that reads no file says it found + # nothing rather than leaving a cut save to wonder whether it ran. + for name in ("tool_evidence", "prompt_copy", "model"): + stage_ended(name, None) + since = len(problems) runtime = contract.runtime if subfolder and source_root is not None: try: @@ -405,20 +467,20 @@ def _checked_against_source( problems.append(str(invalid)) elif runtime is not None and runtime.source_subfolder: runtime = runtime.model_copy(update={"source_subfolder": ""}) + stage_ended("runtime", None) if problems: return _problems(problems, arrived) - return contract.model_copy( - update={ - # With a folder, what it imports from elsewhere is read too, as the other scans do. - "instrumentation": detect_instrumentation( + if instrumentation is None: + # With a folder, what it imports from elsewhere is read too, as the other scans do. + instrumentation = ( + detect_instrumentation( agent_folder(source_root, subfolder), code_the_agent_reaches(source_root, subfolder) if reached is None else reached, ) if source_root and subfolder - else detect_instrumentation(source_root), - "runtime": runtime, - } - ) + else detect_instrumentation(source_root) + ) + return contract.model_copy(update={"instrumentation": instrumentation, "runtime": runtime}) def _save_contract(contract: AgentContract, destination: Path) -> dict[str, Any]: @@ -595,16 +657,16 @@ def _builds_a_sibling(source_root: Path, runtime: Runtime, subfolder: str) -> st def _tools_in_agent_folder( scan_root: Path | None, subfolder: str | None, reached: list[Path] | None = None -) -> list[str]: - """The browser tools this agent uses, which a sibling's aren't. ``reached`` is filled with - the code it reaches outside its folder.""" +) -> tuple[list[str], frozenset[str]]: + """The browser tools this agent uses, which a sibling's aren't, and the limits that cut the + scan. ``reached`` is filled with the code it reaches outside its folder.""" if not scan_root or not subfolder: - return browser_tools_found(scan_root) + return browser_scan(scan_root) try: - return browser_tools_in_agent(scan_root, subfolder, reached) + return browser_scan(scan_root, subfolder, reached) except SubfolderNotFound: # The folder check reports this; the model's reading shouldn't be refused for it too. - return [] + return [], frozenset() def contract_tools( @@ -615,12 +677,22 @@ def contract_tools( scan_root: Path | None = None, require_setup_contract: bool = False, subfolder: str | None = None, + save_checks: Mode | None = None, + save_stand_in: str | None = None, ) -> Any: """A server exposing ``submit_contract``, writing to ``destination`` on acceptance. ``available_evals`` is the platform's own eval catalogue when the job carried one. Absent, the contract simply chooses none, which is what every job did before the catalogue existed. + + ``save_checks`` picks where the checks that read the customer's files run (the default is the + kit's); ``save_stand_in`` names a module whose stages replace the real ones in the child, for + tests. """ + # Here rather than at the top: the checker's child imports this module. + from . import save_checker + from .save_checker import SAVE_DEADLINE, NotInTime, Saved + events_schema = CompletedToolEvents.model_json_schema() events_schema["properties"]["failure_events"]["items"] = ( events_schema.pop("$defs")["CompletedToolFailureEvent"] @@ -649,14 +721,20 @@ def contract_tools( refused = False second_look = SecondLook() found: list[str] | None = None + # Set with found, from the same scan: a later call skips the scan and must still see its cut. + cut: frozenset[str] = frozenset() # Calls in one message check side by side, so the one sent last can finish first; it's the # model's last word, and an earlier call that finishes later mustn't write over it. sent = saved = 0 - # Calls in one message share one scan: each running its own takes that many times longer. - scan: asyncio.Future[list[str]] | None = None - # The code outside the folder the scan found the agent reaches, for the instrumentation - # check, so the save walks the imports once. - reach: list[Path] = [] + # The customer's files are read where a crafted repository can't hold the conversation: the + # checker owns the scan, which calls in one message share, and each save's checks. + checker = save_checker.SaveChecker( + scan_root=scan_root, + source_root=source_root, + subfolder=subfolder, + mode=save_checks, + stand_in=save_stand_in, + ) @tool( "submit_contract", @@ -1476,27 +1554,21 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: receipt = payload.pop(SECOND_LOOK_READ, None) or ( args.get(SECOND_LOOK_READ) if isinstance(args, dict) else None ) - nonlocal refused, found, sent, saved, scan + nonlocal refused, found, cut, sent, saved # Taken before the scan's wait: a call already running when a look went out hasn't read it. looked_before = second_look.given sent += 1 mine = sent if refused or (destination / FAILURE_FILE).is_file(): return _ok(_stop_message(destination)) + # One deadline for everything this call waits on, so waiting on the scan leaves the + # checks less time rather than starting the clock again. + deadline = asyncio.get_running_loop().time() + SAVE_DEADLINE if found is None: - # Run again only a scan that can't give its answer here: one that raised or was - # cancelled, or one on another loop, which this one can't wait on. One that finished - # after every call waiting on it was cancelled still holds its answer. - if ( - scan is None - or scan.get_loop() is not asyncio.get_running_loop() - or (scan.done() and (scan.cancelled() or scan.exception() is not None)) - ): - scan = asyncio.ensure_future( - asyncio.to_thread(_tools_in_agent_folder, scan_root, subfolder, reach) - ) - # Shielded: one call cancelled mustn't cancel the scan the others wait on. - found = await asyncio.shield(scan) + answer = await checker.scan(deadline=deadline) + if isinstance(answer, NotInTime): + return answer.response + found, cut = answer.found, answer.cut # A call running alongside this one may have refused while this one waited. if refused or (destination / FAILURE_FILE).is_file(): return _ok(_stop_message(destination)) @@ -1508,7 +1580,9 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: "job is work on websites or a computer." ] ) - verdict = second_look.verdict(claimed, receipt, found, given_before=looked_before) + verdict = second_look.verdict( + claimed, receipt, found, given_before=looked_before, cut=cut + ) if verdict == "refuse": # Set first: if the write fails, the model must not get a resubmission through. refused = True @@ -1520,7 +1594,7 @@ async def submit_contract(args: dict[str, Any]) -> dict[str, Any]: write_failure_file(agent_type_failure(found), destination) return _ok(_stop_message(destination)) if verdict == "second_look": - return _problems([second_look.issue(claimed, found, _new_receipt())]) + return _problems([second_look.issue(claimed, found, _new_receipt(), cut=cut)]) data_store = payload.get("data_store") if not isinstance(data_store, dict): data_store = {} @@ -1766,23 +1840,26 @@ def rows_for(collection: str) -> list[dict[str, Any]]: return _problems( say + ["If any of these genuinely does not apply, submit again as is."] ) - # The checks read the customer's files, which can take a second on a crafted one, so - # they run off the loop. The write stays here, after a refusal that came meanwhile. - checked = await asyncio.to_thread( - _checked_contract, + # A malformed payload is answered here, without waiting on the checks that read files. + quick = _quick_schema(payload, sorted(payload)) + if quick is not None: + nudged.update(asked_before) + return quick + # The checks read the customer's files, which can take seconds on a crafted one, so they + # run where the deadline can stop them. The write stays here, after a refusal that came + # meanwhile. + checked = await checker.check( payload, - source_root=source_root, require_setup_contract=require_setup_contract, - subfolder=subfolder, - # The scan's answer is for scan_root; another root's code is found again. - reached=reach if scan_root == source_root else None, asked=asked_before, held=held, + deadline=deadline, ) # Whatever this call asked is asked for the calls sent after it. nudged.update(asked_before) - if isinstance(checked, dict): - return checked + if not isinstance(checked, Saved): + # Refused, or not checked in time: either way nothing is saved. + return checked.response if refused or (destination / FAILURE_FILE).is_file(): return _ok(_stop_message(destination)) if mine < saved: @@ -1790,11 +1867,17 @@ def rows_for(collection: str) -> list[dict[str, Any]]: "Not saved: a contract submitted after this one, in the same turn, was saved " "instead. Don't submit this one again." ) - result = _save_contract(checked, destination) + result = _save_contract(checked.contract, destination) # After the write: if it raised, nothing was saved for an earlier call to defer to. saved = mine return result + if scan_root is not None or source_root is not None: + checker.start() + # On the handler, not the server: every copy of the server the session makes keeps the + # handler, so the child ends only when the last of them is gone. + weakref.finalize(submit_contract.handler, checker.close) + save_checker._OPEN.add(checker) return tool_server(name=CONTRACT_SERVER, version="0.1.0", tools=[submit_contract]) diff --git a/tests/harness/_save_stand_ins.py b/tests/harness/_save_stand_ins.py new file mode 100644 index 000000000..1e5d919b9 --- /dev/null +++ b/tests/harness/_save_stand_ins.py @@ -0,0 +1,164 @@ +"""Stages that stand in for the save checks in the save checker's child. + +A patch in the test's own process can't reach the child, so a test names this module and the +child swaps these in after its trial. Each one runs the real check unless told otherwise: + +- a save's stages read what to do from the contract's ``notes``, as ``:[:]`` + words. The ``contract`` stage runs first in every save, so it keeps the words for the rest. + Actions: ``sleep`` (``arg`` seconds, past the deadline by default), ``refuse``, ``raise``, + ``exit`` (the child ends), and ``wait`` (writes ``.-running`` in the source root, then + waits for ``.-go`` there, within the deadline); +- the session's two parts read ``save-stand-in.json`` at the root (instrumentation looks in the + folder it is given and up to three folders above it). ``"scan"``: ``"raise"``, ``"cut"``, + ``"real"``, or ``{"after": , "sleep": }``, which writes ``.scan-waiting``, + waits for that file at the root, then sleeps. A scan told anything writes ``.scan-done`` once + it answers. ``"instrumentation"``: ``"raise"``, ``{"hold": }``, or + ``{"value": }``. +""" + +from __future__ import annotations + +import json +import os +import time +from pathlib import Path +from typing import Any + +from fi.alk.harness import instrumentation_evidence, source_tool_evidence, tools + +PAST_THE_DEADLINE = 3.0 +SESSION_FILE = "save-stand-in.json" +# Long enough for any test's signal, and short of the child's session bound. +_LONGEST_WAIT = 20.0 + +_real_scan = tools._tools_in_agent_folder +_real_instrumentation = instrumentation_evidence.detect_instrumentation +_real_validate = tools.validate_contract +_real_tool_evidence = source_tool_evidence.tool_evidence_problems +_real_copy = tools._copy_prompt +_real_model = tools.model_problems +_real_sibling = tools._builds_a_sibling + +# The words of the save now running; the child runs one save at a time. +_said: dict[str, tuple[str, str]] = {} + + +def _words(notes: str) -> dict[str, tuple[str, str]]: + said = {} + for word in notes.split(): + stage, _, rest = word.partition(":") + action, _, arg = rest.partition(":") + if stage and action: + said[stage] = (action, arg) + return said + + +def _act(stage: str, root: Path | None) -> str | None: + """Carry out what the save said for ``stage``; ``"refuse"`` when it should refuse.""" + action, arg = _said.get(stage, ("", "")) + if action == "sleep": + time.sleep(float(arg) if arg else PAST_THE_DEADLINE) + elif action == "raise": + raise RuntimeError(f"stand-in {stage} raised") + elif action == "exit": + os._exit(3) + elif action == "wait" and root is not None: + (root / f".{stage}-running").write_text("") + go = root / f".{stage}-go" + until = time.monotonic() + (float(arg) if arg else 1.2) + while not go.exists() and time.monotonic() < until: + time.sleep(0.005) + elif action == "refuse": + return "refuse" + return None + + +def _session_said(root: Path | None, part: str) -> Any: + folder = root + for _ in range(4): + if folder is None: + return None + found = folder / SESSION_FILE + if found.is_file(): + return json.loads(found.read_text(encoding="utf-8")).get(part) + folder = folder.parent + return None + + +def scan(scan_root, subfolder, reached=None): + said = _session_said(Path(scan_root) if scan_root else None, "scan") + if said == "raise": + raise OSError("stand-in scan raised") + if said == "cut": + return [], frozenset({"scan_total"}) + if isinstance(said, dict): + # Said first, so a test starts its clock only once the session job is here. + (Path(scan_root) / ".scan-waiting").write_text("") + go = Path(scan_root) / said["after"] + until = time.monotonic() + _LONGEST_WAIT + while not go.exists() and time.monotonic() < until: + time.sleep(0.005) + time.sleep(float(said.get("sleep", 0))) + answer = _real_scan(scan_root, subfolder, reached) + if said is not None: + (Path(scan_root) / ".scan-done").write_text("") + return answer + + +def instrumentation(source_root, reached=()): + said = _session_said(Path(source_root) if source_root else None, "instrumentation") + if said == "raise": + raise OSError("stand-in instrumentation raised") + if isinstance(said, dict) and "hold" in said: + time.sleep(float(said["hold"])) + if isinstance(said, dict) and "value" in said: + return str(said["value"]) + return _real_instrumentation(source_root, reached) + + +def validate(contract): + _said.clear() + _said.update(_words(contract.notes or "")) + problems = _real_validate(contract) + if _act("contract", None) == "refuse": + problems.append("stand-in: contract refused") + return problems + + +def tool_evidence(contract, source, subfolder=None): + problems = _real_tool_evidence(contract, source, subfolder) + if _act("tool_evidence", Path(source)) == "refuse": + problems.append("stand-in: tool_evidence refused") + return problems + + +def copy_prompt(contract, source_root): + problems = _real_copy(contract, source_root) + if _act("prompt_copy", Path(source_root)) == "refuse": + problems.append("stand-in: prompt_copy refused") + return problems + + +def model(contract, source_root): + problems = _real_model(contract, source_root) + if _act("model", Path(source_root)) == "refuse": + problems.append("stand-in: model refused") + return problems + + +def sibling(source_root, runtime, subfolder): + refusal = _real_sibling(source_root, runtime, subfolder) + if _act("runtime", Path(source_root)) == "refuse": + return "stand-in: runtime refused" + return refusal + + +STAGES = { + ("tools", "_tools_in_agent_folder"): scan, + ("instrumentation_evidence", "detect_instrumentation"): instrumentation, + ("tools", "validate_contract"): validate, + ("source_tool_evidence", "tool_evidence_problems"): tool_evidence, + ("tools", "_copy_prompt"): copy_prompt, + ("tools", "model_problems"): model, + ("tools", "_builds_a_sibling"): sibling, +} diff --git a/tests/harness/conftest.py b/tests/harness/conftest.py index 52ccb926c..35d6d2d60 100644 --- a/tests/harness/conftest.py +++ b/tests/harness/conftest.py @@ -1,8 +1,71 @@ import hashlib import json +import multiprocessing +import os +import time import pytest +# A killed child is reaped by the fork server, a moment after its parent's join returns. +_REAP_SECONDS = 10.0 + + +@pytest.fixture(scope="session", autouse=True) +def _save_checker_fork_server(): + """Starts the fork server once per test process, so no test pays its start in a deadline.""" + try: + from fi.alk.harness.forkserver_preload import PRELOAD + from fi.alk.harness.save_checker import default_mode + except ImportError: + yield + return + if default_mode() == "child": + try: + context = multiprocessing.get_context("forkserver") + context.set_forkserver_preload(PRELOAD) + warm = context.Process(target=time.sleep, args=(0,), daemon=True) + warm.start() + warm.join(120) + except Exception: + # Only a warm-up: a test that starts a child starts the fork server itself. + pass + yield + + +def _alive(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +@pytest.fixture(autouse=True) +def _save_checkers_closed(): + """Closes every save checker a test left open, and fails it if one of their children lives. + + Only the pids those checkers started are checked, not the fork server's or the folder scan's. + """ + yield + try: + from fi.alk.harness import save_checker + except ImportError: + return + checkers = list(save_checker._OPEN) + pids = [pid for checker in checkers for pid in checker.pids] + for checker in checkers: + checker.close() + save_checker._OPEN.discard(checker) + until = time.monotonic() + _REAP_SECONDS + alive = [pid for pid in pids if _alive(pid)] + while alive and time.monotonic() < until: + time.sleep(0.05) + alive = [pid for pid in alive if _alive(pid)] + if alive: + pytest.fail(f"save checker children still running after the test: {alive}") + @pytest.fixture def resolved_compose_images(monkeypatch): diff --git a/tests/harness/test_agent_reading.py b/tests/harness/test_agent_reading.py index 63e9fd4e0..40a586d5c 100644 --- a/tests/harness/test_agent_reading.py +++ b/tests/harness/test_agent_reading.py @@ -1865,6 +1865,8 @@ def test_a_cancelled_contract_save_writes_nothing(tmp_path, monkeypatch, where, only after the scan, and the refusing call reaches its write with no wait after the scan. So the scan is its only wait (``scan-browser``). """ + # It patches checks the save would otherwise run in a child, so they run in this process. + monkeypatch.setenv("ALK_SAVE_CHECKS", "thread") subfolder = SCRAPER if where == "scan-browser" else SUPPORT spec, output = _reading_spec(tmp_path, _job(subfolder=subfolder)) tool = _submit_contract(spec) @@ -1920,6 +1922,8 @@ def test_the_readings_gemini_tools_run_on_its_loop(tmp_path, monkeypatch): So the reading's last cancel reaches it. ADK's tool_thread_pool_config would move it to a thread's own loop, but only in live mode in the locked 2.7.1, which a reading never uses. """ + # It patches checks the save would otherwise run in a child, so they run in this process. + monkeypatch.setenv("ALK_SAVE_CHECKS", "thread") from google.adk.models.google_llm import Gemini from google.adk.models.llm_response import LlmResponse from google.adk.runners import Runner diff --git a/tests/harness/test_agent_type_refusal.py b/tests/harness/test_agent_type_refusal.py index 1faa89df8..209294e26 100644 --- a/tests/harness/test_agent_type_refusal.py +++ b/tests/harness/test_agent_type_refusal.py @@ -113,8 +113,8 @@ def _submit_tool(server): return next(one for one in server.tools if one.name == "submit_contract") -def _submitter(destination: Path, scan_root: Path | None = None): - tool = _submit_tool(contract_tools(destination, scan_root=scan_root)) +def _submitter(destination: Path, scan_root: Path | None = None, save_checks=None): + tool = _submit_tool(contract_tools(destination, scan_root=scan_root, save_checks=save_checks)) return lambda payload: asyncio.run(tool.handler(payload)) @@ -614,9 +614,10 @@ def odd(path, *args, **kwargs): monkeypatch.setattr(agent_type_check, "read_module", odd) with caplog.at_level("WARNING", logger=agent_type_check.__name__): assert browser_tools_found(tmp_path) == ["playwright"] - [record] = caplog.records - assert record.getMessage() == "agent type scan skipped a file: LookupError" - assert record.exc_info is None + skipped, cut = caplog.records + assert skipped.getMessage() == "agent type scan skipped a file: LookupError" + assert skipped.exc_info is None + assert cut.getMessage() == "agent type scan cut by skipped 1" def test_a_skip_logs_its_type_once_and_its_path_at_debug_only_with_no_trace( @@ -636,7 +637,7 @@ def odd(path, *args, **kwargs): loud = [record for record in caplog.records if record.levelno >= 20] assert [record.getMessage() for record in loud] == [ "agent type scan skipped a file: LookupError", - "agent type scan skipped 2 files", + "agent type scan cut by skipped 2", ] assert not any( "SOURCE-LINE" in record.getMessage() or "odd" in record.getMessage() or record.exc_info @@ -658,12 +659,12 @@ def odd(path, *args, **kwargs): @pytest.mark.skipif(sys.version_info < (3, 12), reason="3.11's tokenizer reads a NUL byte") def test_a_file_with_a_nul_byte_is_left_unread_without_stopping_the_scan(tmp_path, caplog): # Through the shared reader a NUL byte is a file Python can't parse, an expected answer, not - # an error: it's left unread and nothing is logged. A file naming nothing isn't even read. + # an error: it's left unread and only the cut is logged. A file naming nothing isn't even read. (tmp_path / "a_blob.py").write_bytes(b" x = 1\n\x00") (tmp_path / "b_blob.py").write_bytes(b"import playwright\n\x00") with caplog.at_level("WARNING", logger=agent_type_check.__name__): assert browser_tools_found(tmp_path) == [] - assert caplog.records == [] + assert [r.getMessage() for r in caplog.records] == ["agent type scan cut by parse_failed 1"] @pytest.mark.skipif(sys.version_info < (3, 12), reason="3.11's tokenizer reads a NUL byte") @@ -673,7 +674,10 @@ def test_a_nul_byte_the_tokenizer_breaks_on_reaches_no_warning(tmp_path, caplog) (tmp_path / "agent.py").write_text("import playwright\n", encoding="utf-8") with caplog.at_level("DEBUG", logger=agent_type_check.__name__): assert browser_tools_found(tmp_path) == ["playwright"] - assert [record for record in caplog.records if record.levelno >= 20] == [] + # The tokenizer's fault reaches no WARNING of its own. The files name a tool and can't be + # parsed, so they cut the scan, and that line holds names and counts only. + loud = [record.getMessage() for record in caplog.records if record.levelno >= 20] + assert loud == ["agent type scan cut by parse_failed 30"] assert "SOURCE-LINE" not in caplog.text @@ -1975,12 +1979,12 @@ def test_unknown_modality_gets_a_problem_not_a_save(tmp_path, plain_repo): def test_the_scan_runs_off_the_event_loop(tmp_path, monkeypatch): threads = [] - def scan(_root): + def scan(_root, subfolder=None, reached=None): threads.append(threading.current_thread()) - return [] + return [], frozenset() - monkeypatch.setattr(tools, "browser_tools_found", scan) - _submitter(tmp_path / "out", tmp_path)(_reading("chat")) + monkeypatch.setattr(tools, "browser_scan", scan) + _submitter(tmp_path / "out", tmp_path, "thread")(_reading("chat")) assert threads and threads[0] is not threading.main_thread() @@ -1988,12 +1992,12 @@ def scan(_root): def test_a_refusal_during_the_scan_stops_this_call_too(tmp_path, monkeypatch, modality): output = tmp_path / "out" - def scan(_root): + def scan(_root, subfolder=None, reached=None): write_failure_file(agent_type_failure(["playwright"]), output) - return [] + return [], frozenset() - monkeypatch.setattr(tools, "browser_tools_found", scan) - answer = _submitter(output, tmp_path)(_reading(modality)) + monkeypatch.setattr(tools, "browser_scan", scan) + answer = _submitter(output, tmp_path, "thread")(_reading(modality)) assert not (output / "contract.json").exists() # The stop itself, not a second look: the reading was refused while the scan ran. A browser # reading would otherwise be asked for a second look before the write's own check. @@ -2770,13 +2774,13 @@ def test_conversation_with_no_tool_saves_with_no_second_look(tmp_path, plain_rep def _counted_scans(monkeypatch) -> list[Path]: scans: list[Path] = [] - real_scan = tools.browser_tools_found + real_scan = tools.browser_scan - def counted(root): + def counted(root, subfolder=None, reached=None): scans.append(root) - return real_scan(root) + return real_scan(root, subfolder, reached) - monkeypatch.setattr(tools, "browser_tools_found", counted) + monkeypatch.setattr(tools, "browser_scan", counted) return scans @@ -2785,7 +2789,7 @@ def test_one_scan_and_one_look_when_an_answer_comes_back_without_its_value( ): output = tmp_path / "out" scans = _counted_scans(monkeypatch) - submit = _submitter(output, browser_repo) + submit = _submitter(output, browser_repo, "thread") assert "second-look" in _text(submit(_reading("chat"))) # An answer with the value stands, even when the contract is then sent back for another fix. rejected = submit(_reading("chat", real_use_cases=[], second_look_read=_READ)) @@ -2809,7 +2813,7 @@ def test_a_different_answer_without_the_value_after_an_answer_gets_a_look(tmp_pa def test_calls_waiting_on_the_first_scan_share_it(tmp_path, plain_repo, monkeypatch): scans = _counted_scans(monkeypatch) - tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=plain_repo)) + tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=plain_repo, save_checks="thread")) async def first_message(): return await asyncio.gather(*(tool.handler(_reading("browser")) for _ in range(3))) @@ -2823,13 +2827,13 @@ def test_a_call_cancelled_while_waiting_on_the_scan_leaves_it_to_the_others( ): started, release = threading.Event(), threading.Event() - def slow(_root): + def slow(_root, subfolder=None, reached=None): started.set() release.wait(10) - return [] + return [], frozenset() - monkeypatch.setattr(tools, "browser_tools_found", slow) - tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=plain_repo)) + monkeypatch.setattr(tools, "browser_scan", slow) + tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=plain_repo, save_checks="thread")) async def first_message(): cancelled = asyncio.ensure_future(tool.handler(_reading("browser"))) @@ -2844,16 +2848,16 @@ async def first_message(): def test_a_scan_that_fails_is_run_again_by_the_next_call(tmp_path, plain_repo, monkeypatch): - outcomes = iter([OSError("gone"), []]) + outcomes = iter([OSError("gone"), ([], frozenset())]) - def scan(_root): + def scan(_root, subfolder=None, reached=None): outcome = next(outcomes) if isinstance(outcome, Exception): raise outcome return outcome - monkeypatch.setattr(tools, "browser_tools_found", scan) - submit = _submitter(tmp_path / "out", plain_repo) + monkeypatch.setattr(tools, "browser_scan", scan) + submit = _submitter(tmp_path / "out", plain_repo, "thread") with pytest.raises(OSError, match="gone"): submit(_reading("chat")) assert not submit(_reading("chat")).get("is_error") @@ -2862,16 +2866,16 @@ def scan(_root): def test_a_scan_that_fails_is_run_again_by_the_next_call_in_the_same_message( tmp_path, plain_repo, monkeypatch ): - outcomes = iter([OSError("gone"), []]) + outcomes = iter([OSError("gone"), ([], frozenset())]) - def scan(_root): + def scan(_root, subfolder=None, reached=None): outcome = next(outcomes) if isinstance(outcome, Exception): raise outcome return outcome - monkeypatch.setattr(tools, "browser_tools_found", scan) - tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=plain_repo)) + monkeypatch.setattr(tools, "browser_scan", scan) + tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=plain_repo, save_checks="thread")) async def one_message(): with pytest.raises(OSError, match="gone"): @@ -2887,14 +2891,14 @@ def test_a_scan_that_finished_after_its_only_caller_was_cancelled_is_not_run_aga started, release = threading.Event(), threading.Event() scans: list[Path] = [] - def slow(root): + def slow(root, subfolder=None, reached=None): scans.append(root) started.set() release.wait(10) - return [] + return [], frozenset() - monkeypatch.setattr(tools, "browser_tools_found", slow) - tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=plain_repo)) + monkeypatch.setattr(tools, "browser_scan", slow) + tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=plain_repo, save_checks="thread")) async def cancelled_then_next(): cancelled = asyncio.ensure_future(tool.handler(_reading("browser"))) @@ -2913,15 +2917,15 @@ def test_a_scan_left_running_on_a_closed_loop_is_started_again(tmp_path, plain_r started, release = threading.Event(), threading.Event() scans: list[Path] = [] - def scan(root): + def scan(root, subfolder=None, reached=None): scans.append(root) if len(scans) == 1: started.set() release.wait(10) - return [] + return [], frozenset() - monkeypatch.setattr(tools, "browser_tools_found", scan) - tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=plain_repo)) + monkeypatch.setattr(tools, "browser_scan", scan) + tool = _submit_tool(contract_tools(tmp_path / "out", scan_root=plain_repo, save_checks="thread")) closed = asyncio.new_event_loop() try: closed.create_task(tool.handler(_reading("browser"))) @@ -3685,9 +3689,16 @@ async def beat(): worst[0], last = max(worst[0], now - last), now beating = asyncio.create_task(beat()) - started = time.perf_counter() - result = await server.tools[0].handler(reading) - waited = time.perf_counter() - started + # A save made while the session's checks still run answers "not checked in time" (on a + # loaded host they can outlast the deadline), and the model writes again: so does this. + waited = 0.0 + for _ in range(60): + started = time.perf_counter() + result = await server.tools[0].handler(reading) + waited = max(waited, time.perf_counter() - started) + if not _text(result).startswith("not checked in time"): + break + await asyncio.sleep(0.5) done.set() await beating return result, worst[0], waited @@ -6698,3 +6709,94 @@ def test_a_decode_total_spent_exactly_doesnt_say_the_walk_stopped(tmp_path, monk found, stopped = _save_walk(repo) assert not stopped and repo / "common" / "web.py" in found + + +# The scan and the save in the save checker's child, through every way the server is held + + +def _checker_of(handler): + import inspect + + return inspect.getclosurevars(handler).nonlocals["checker"] + + +def test_a_scan_that_raises_twice_in_the_child_counts_as_cut(tmp_path, plain_repo): + from fi.alk.harness.save_checker import SessionChecksFailed + + (plain_repo / "save-stand-in.json").write_text(json.dumps({"scan": "raise"})) + tool = _submit_tool( + contract_tools( + tmp_path / "out", + scan_root=plain_repo, + save_checks="child", + save_stand_in="_save_stand_ins", + ) + ) + + async def calls(): + with pytest.raises(SessionChecksFailed): + await tool.handler(_reading("chat")) + return await tool.handler(_reading("chat")) + + answer = asyncio.run(calls()) + + # A scan that never answered can't say there is no browser tool: the reading gets a look. + assert "second-look" in _text(answer) + assert "stopped before reading all of the agent's code" in _text(answer) + assert not (tmp_path / "out" / "contract.json").exists() + assert _checker_of(tool.handler).ran_in_child == 0 + + +def test_the_hosted_builders_copy_of_the_server_saves_in_the_child( + tmp_path, plain_repo, monkeypatch +): + import gc + + from fi.alk.harness.builder import open_for_building + + monkeypatch.delenv("ALK_SAVE_CHECKS", raising=False) + output = tmp_path / "out" + stage = open_for_building(None, out=output, source=RepoSource(name="fixture", root=plain_repo)) + # The hosted chat puts a copy of each server in place of the one the stage opened with. + _runtime(tmp_path)._guard(stage.spec) + gc.collect() + tool = _submit_tool(_server_of(stage)) + + async def calls(): + # The first is asked for the prompt's lines; the second is accepted. + await tool.handler(_reading("chat")) + return await tool.handler(_reading("chat")) + + answer = asyncio.run(calls()) + + assert not answer.get("is_error"), answer + assert load(output).modality == "chat" + assert _checker_of(tool.handler).ran_in_child == 1 + + +def test_a_stage_opened_again_leaves_no_child_behind(tmp_path, plain_repo, monkeypatch): + import gc + import multiprocessing + + from fi.alk.harness.builder import open_for_building + + monkeypatch.delenv("ALK_SAVE_CHECKS", raising=False) + source = RepoSource(name="fixture", root=plain_repo) + before = {child.pid for child in multiprocessing.active_children()} + first = open_for_building(None, out=tmp_path / "out", source=source) + handler = _submit_tool(_server_of(first)).handler + + async def calls(handler): + await handler(_reading("chat")) + return await handler(_reading("chat")) + + assert not asyncio.run(calls(handler)).get("is_error") + started = {child.pid for child in multiprocessing.active_children()} - before + assert started + + del first, handler + again = open_for_building(None, out=tmp_path / "out", source=source) + gc.collect() + + assert not started & {child.pid for child in multiprocessing.active_children()} + assert _submit_tool(_server_of(again)) diff --git a/tests/harness/test_browser_scan_cut.py b/tests/harness/test_browser_scan_cut.py new file mode 100644 index 000000000..7f8c26e02 --- /dev/null +++ b/tests/harness/test_browser_scan_cut.py @@ -0,0 +1,866 @@ +"""A browser scan that leaves code unread or unparsed says so, and the reading gets a second look. + +Finding nothing in code the scan didn't read isn't finding nothing: padding read first, or an +honest repository larger than the limits, would otherwise hide a plain ``import playwright``. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import os +from pathlib import Path + +import pytest + +from fi.alk.harness import agent_type_check, python_source, tools +from fi.alk.harness.agent_type_check import ( + SecondLook, + browser_scan, + browser_tools_found, + browser_tools_in_agent, + decide, +) +from fi.alk.harness.failure_file import FAILURE_FILE +from fi.alk.harness.python_source import LARGEST_PARSED_MODULE +from fi.alk.harness.tools import contract_tools +from fi.alk.harness.understand import load + +_READ = "read-it" +_NOT_ACCEPTED = "Not accepted. Fix all of these and call submit_contract again:\n - " +_RECEIPT = f' When you submit again, set second_look_read to "{_READ}", which shows you read this.' +_CUT_CHAT = ( + "second-look: you recorded modality {claimed}, and the code check stopped before reading all " + "of the agent's code (it holds more than the check reads, or a file it couldn't read), so it " + "can't say whether a browser, crawling or computer-use tool is used. A chat or voice agent " + "can use one in its tests or a helper script. If a person comes to this agent for a " + "conversation, submit again as is. If its main job is carrying out work on websites or a " + "computer, submit again as browser, which stops this build as not supported. Decide this " + "from the source; don't ask anyone." +) +_CUT_BROWSER = ( + "second-look: you recorded modality browser, which stops this build as not supported, and " + "the code check stopped before reading all of the agent's code (it holds more than the check " + "reads, or a file it couldn't read), so it can't say whether a browser, crawling or " + "computer-use tool is used. A chat or voice agent can use one in its tests or a helper " + "script. Does a person come to this agent for a conversation, or to have it carry out work on " + "websites or a computer? For a conversation, submit again as chat or voice. If you're sure, " + "submit again as browser. Decide this from the source; don't ask anyone." +) +_FOUND_CHAT = ( + "second-look: you recorded modality {claimed}, and the source uses playwright. If that's a " + "helper and a person comes to this agent for a conversation, submit again as is. If its main " + "job is carrying out work on websites or a computer, submit again as browser, which stops " + "this build as not supported. Decide this from the source; don't ask anyone." +) +_FOUND_BROWSER = ( + "second-look: you recorded modality browser, which stops this build as not supported, and " + "the source uses playwright. A chat or voice agent can list one for its tests or use one in " + "a helper script. Does a person come to this agent for a conversation, or to have it carry " + "out work on websites or a computer? For a conversation, submit again as chat or voice. If " + "its main job is that work, submit again as browser. Decide this from the source; don't ask " + "anyone." +) +_NOTHING_FOUND_BROWSER = ( + "second-look: you recorded modality browser, which stops this build as not supported, and " + "no browser, crawling or computer-use tool was found in the source. Does a person come to " + "this agent for a conversation, or to have it carry out work on websites or a computer? For " + "a conversation, submit again as chat or voice. If you're sure, submit again as browser. " + "Decide this from the source; don't ask anyone." +) +_FOUND_MESSAGE = ( + "This agent's main job is driving a browser, operating a computer or crawling sites " + "(found: playwright). Only voice and chat agents are supported for now." +) +_READ_MESSAGE = ( + "This agent's main job was read as driving a browser, operating a computer or crawling " + "sites. Only voice and chat agents are supported for now." +) +# Over the size a .py is read at, so never read; it names a tool, so leaving it unread cuts. +_OVERSIZED = "# playwright\n".ljust(LARGEST_PARSED_MODULE + 1, "#") +# A data module of prompt strings over that size, about 265 KB, naming no tool. +_DATA_MODULE = ( + "PROMPTS = [\n" + ' "Tell me about the weather today.",\n' * 6_800 + "]\n" +) + + +@pytest.fixture(autouse=True) +def _nothing_reaches_a_network(monkeypatch): + for name in ( + "FI_BASE_URL", + "OTEL_EXPORTER_OTLP_ENDPOINT", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + ): + monkeypatch.setenv(name, "http://127.0.0.1:9") + monkeypatch.delenv("FI_API_KEY", raising=False) + monkeypatch.delenv("FI_SECRET_KEY", raising=False) + + +@pytest.fixture(autouse=True) +def _a_known_receipt(monkeypatch): + monkeypatch.setattr(tools, "_new_receipt", lambda: _READ) + + +def _tree(root: Path, files: dict[str, str | bytes]) -> Path: + for name, body in files.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(body, bytes): + path.write_bytes(body) + else: + path.write_text(body, encoding="utf-8") + return root + + +def _reading(modality: str | None = "chat", **fields): + payload = { + "agent": "fixture", + "one_liner": "Answers questions.", + "real_use_cases": ["Answer a question"], + "tools": [{"name": "lookup", "args": ["query"]}], + "hard_constraints": ["Stay polite"], + "system_prompt_excerpt": "You answer questions.", + **fields, + } + if modality is not None: + payload["modality"] = modality + return payload + + +def _submitter( + destination: Path, scan_root: Path, subfolder: str | None = None, save_checks=None +): + server = contract_tools( + destination, scan_root=scan_root, subfolder=subfolder, save_checks=save_checks + ) + tool = next(one for one in server.tools if one.name == "submit_contract") + return lambda payload: asyncio.run(tool.handler(payload)) + + +def _text(answer) -> str: + return answer["content"][0]["text"] + + +def _look(text: str) -> str: + return _NOT_ACCEPTED + text + _RECEIPT + + +def _refused_with(output: Path) -> str: + assert not (output / "contract.json").exists() + return json.loads((output / FAILURE_FILE).read_text(encoding="utf-8"))["message"] + + +def _raise_for_agent(monkeypatch): + real = agent_type_check.read_module + + def odd(path, *args, **kwargs): + if path.name == "agent.py": + raise LookupError("an error no reader expects") + return real(path, *args, **kwargs) + + monkeypatch.setattr(agent_type_check, "read_module", odd) + + +# Totals set low, so a few bytes reach each limit as a large repository would. +_LIMITS = [ + pytest.param( + "file_size", + {"agent.py": "import playwright\n".ljust(LARGEST_PARSED_MODULE + 1, "#")}, + {}, + id="file_size-python", + ), + pytest.param( + "file_size", + {"agent.js": "const tool = 'computer-use-preview'; // past forty bytes\n"}, + {(agent_type_check, "_LARGEST_CODE_FILE"): 40}, + id="file_size-script", + ), + pytest.param( + "scan_total", + {"agent.py": "import playwright\n"}, + {(agent_type_check, "_LARGEST_SCAN"): 10}, + id="scan_total", + ), + pytest.param( + "script_total", + {"agent.js": "const tool = 'computer-use-preview'\n"}, + {(agent_type_check, "_LARGEST_SCRIPT_SCAN"): 10}, + id="script_total", + ), + pytest.param( + "python_total", + {"agent.py": "import playwright\n"}, + {(agent_type_check, "_LARGEST_PYTHON_SCAN"): 10}, + id="python_total", + ), + pytest.param( + "parse_total", + {"agent.py": "import playwright\n"}, + {(python_source, "LARGEST_PARSED_TOTAL"): 10}, + id="parse_total", + ), + pytest.param( + "parse_failed", + {"agent.py": "import playwright\ndef (\n"}, + {}, + id="parse_failed", + ), + pytest.param( + "token_total", + {"agent.py": "TOOL = 'computer_use_preview'\n"}, + {(python_source, "LARGEST_TOKENIZED_TOTAL"): 10}, + id="token_total-total", + ), + pytest.param( + "token_total", + {"agent.py": "TOOL = 'computer_use_preview'\n"}, + {(python_source, "LARGEST_TOKENIZED_WORK"): 1}, + id="token_total-work", + ), + pytest.param( + "decode_total", + {"agent.py": b"# coding: utf-7\nimport playwright\n"}, + {(python_source, "LARGEST_ESCAPED_TOTAL"): 0}, + id="decode_total", + ), + pytest.param( + "search_total", + {"agent.py": b"# coding: idna\nimport playwright\n"}, + {(agent_type_check, "_LARGEST_RAW_SEARCH"): 0}, + id="search_total", + ), + pytest.param( + "skipped", {"agent.py": "import playwright\n"}, _raise_for_agent, id="skipped" + ), + pytest.param( + "package_unreadable", + {"package.json": "[" * 100_000}, + {}, + id="package_unreadable", + ), +] + + +@pytest.mark.parametrize("limit, files, totals", _LIMITS) +def test_each_limit_cuts_the_scan(tmp_path, monkeypatch, limit, files, totals): + _tree(tmp_path, files) + if callable(totals): + totals(monkeypatch) + else: + for (module, name), value in totals.items(): + monkeypatch.setattr(module, name, value) + names, cut = browser_scan(tmp_path) + assert cut == frozenset({limit}) + assert decide("chat", names, looked_again=False, cut=cut) == "second_look" + + +@pytest.mark.parametrize( + "files, spent, names", + [ + # Read and charged, the padding would spend the total before the file that imports. + pytest.param( + { + **{f"a_pad_{at}.py": "x = 1\n" * (126 * 1024 // 6) for at in range(3)}, + "z_agent.py": "import playwright\n", + }, + False, + ["playwright"], + id="padding", + ), + pytest.param({"helper.py": "x = 1\n"}, True, [], id="needle-free"), + pytest.param( + {"agent.py": b"import playwright\n\xff\n"}, True, [], id="undecodable" + ), + ], +) +def test_a_file_that_cant_matter_is_no_cut(tmp_path, monkeypatch, files, spent, names): + _tree(tmp_path, files) + if spent: + monkeypatch.setattr(python_source, "LARGEST_PARSED_TOTAL", 0) + assert browser_scan(tmp_path) == (names, frozenset()) + + +@pytest.mark.parametrize("claimed", ["chat", "voice", "", "browser"]) +@pytest.mark.parametrize("found", [[], ["playwright"]]) +@pytest.mark.parametrize("looked_again", [False, True]) +def test_nothing_cut_is_todays_verdict_and_text(claimed, found, looked_again): + browser = claimed == "browser" + if browser: + today = "refuse" if looked_again else "second_look" + else: + today = "second_look" if found and not looked_again else "keep" + assert decide(claimed, found, looked_again=looked_again) == today + assert decide(claimed, found, looked_again=looked_again, cut=frozenset()) == today + if today != "second_look": + return + if browser: + text = _FOUND_BROWSER if found else _NOTHING_FOUND_BROWSER + else: + text = _FOUND_CHAT.format(claimed=claimed or "chat") + assert SecondLook.prompt(claimed, found) == text + assert SecondLook.prompt(claimed, found, cut=frozenset()) == text + assert SecondLook().issue(claimed, found, _READ) == text + _RECEIPT + + +@pytest.mark.parametrize( + "files, claimed, text", + [ + ( + {"agent.py": "import playwright\n"}, + "chat", + _FOUND_CHAT.format(claimed="chat"), + ), + ({"agent.py": "import playwright\n"}, "browser", _FOUND_BROWSER), + ({"agent.py": "import openai\n"}, "browser", _NOTHING_FOUND_BROWSER), + ], +) +def test_an_uncut_scan_gives_todays_look(tmp_path, files, claimed, text): + repo = _tree(tmp_path / "source", files) + assert browser_scan(repo)[1] == frozenset() + assert _text(_submitter(tmp_path / "out", repo)(_reading(claimed))) == _look(text) + + +def test_an_uncut_chat_reading_of_plain_code_is_saved_without_a_look(tmp_path): + repo = _tree(tmp_path / "source", {"agent.py": "import openai\n"}) + answer = _submitter(tmp_path / "out", repo)(_reading("chat")) + assert not answer.get("is_error"), answer + assert load(tmp_path / "out").modality == "chat" + + +def _cut_repo(root: Path, **more: str) -> Path: + return _tree(root, {"agent.py": "import openai\n", "big.py": _OVERSIZED, **more}) + + +@pytest.mark.parametrize( + "steps", + [ + pytest.param([("chat", None, "look"), ("chat", _READ, "saved")], id="chat"), + pytest.param([("voice", None, "look"), ("voice", _READ, "saved")], id="voice"), + pytest.param([(None, None, "look"), (None, _READ, "saved")], id="no-modality"), + pytest.param( + [("chat", None, "look"), ("browser", _READ, "refused")], id="then-browser" + ), + pytest.param( + [ + ("web", None, "problem"), + ("chat", None, "look"), + ("chat", _READ, "saved"), + ], + id="unknown-modality-first", + ), + ], +) +def test_a_cut_chat_reading_gets_one_look(tmp_path, steps): + output = tmp_path / "out" + submit = _submitter(output, _cut_repo(tmp_path / "source")) + for modality, receipt, then in steps: + extra = {} if receipt is None else {"second_look_read": receipt} + answer = submit(_reading(modality, **extra)) + if then == "problem": + assert answer["is_error"] + assert "'web' is not a modality" in _text(answer) + elif then == "look": + assert _text(answer) == _look(_CUT_CHAT.format(claimed=modality or "chat")) + elif then == "saved": + assert not answer.get("is_error"), answer + assert load(output).modality == (modality or "chat") + assert not (output / FAILURE_FILE).exists() + else: + assert _refused_with(output) == _READ_MESSAGE + + +def test_a_cut_reading_gets_a_look_on_the_subfolder_route(tmp_path): + source = tmp_path / "source" + _cut_repo(source / "agents" / "a") + output = tmp_path / "out" + submit = _submitter(output, source, "agents/a") + answer = submit(_reading("chat")) + assert _text(answer) == _look(_CUT_CHAT.format(claimed="chat")) + + +def test_a_package_json_that_isnt_a_file_is_no_cut(tmp_path): + repo = _tree(tmp_path / "source", {"agent.py": "import openai\n"}) + (repo / "package.json").symlink_to(repo / "nowhere.json") + assert browser_scan(repo) == ([], frozenset()) + + +def test_a_large_data_module_is_no_cut(tmp_path): + # A big data module beside code that imports no tool. + assert len(_DATA_MODULE.encode()) > 260 * 1024 + repo = _tree( + tmp_path / "a", {"main.py": "import openai\n", "prompts_data.py": _DATA_MODULE} + ) + assert browser_scan(repo) == ([], frozenset()) + output = tmp_path / "out" + answer = _submitter(output, repo)(_reading("chat")) + assert not answer.get("is_error"), answer + assert load(output).modality == "chat" + + +@pytest.mark.parametrize( + "files, totals", + [ + pytest.param( + {"big.py": "#" * (LARGEST_PARSED_MODULE + 1) + "\nimport playwright\n"}, + {}, + id="named-past-the-read-limit", + ), + pytest.param( + {"big.py": "#" * (3 * 1024 * 1024) + "\nimport playwright\n"}, + {}, + id="named-3-mib-in", + ), + # Exactly the per-file bound: the needle ends at its last byte. + pytest.param( + { + "big.py": "#" * (agent_type_check._LARGEST_UNREAD_SEARCHED - 19) + + "\nimport playwright\n" + }, + {}, + id="named-at-the-bound", + ), + # The toolkit's import holds no lowercase `playwright`. + pytest.param( + { + "big.py": "x = 1\n" * 30_000 + + "from langchain_community.agent_toolkits import PlayWrightBrowserToolkit\n" + }, + {}, + id="toolkit", + ), + # Python refuses the coding line until the codec is registered at run time. + pytest.param( + {"big.py": b"# coding: nonesuch\n" + _DATA_MODULE.encode()}, + {}, + id="refused-coding", + ), + pytest.param( + {"big.py": "x = 1\n" * 30_000 + "TOOL = 'computer_use_preview'\n"}, + {}, + id="marker", + ), + # Python reads it as `import playwright`; its bytes name nothing. + pytest.param( + { + "big.py": b"# coding: utf-7\nimport +AHA-laywright\n" + + _DATA_MODULE.encode() + }, + {}, + id="utf-7", + ), + pytest.param( + { + "big.py": b"# coding: unicode_escape\nimport \\x70laywright\n" + + _DATA_MODULE.encode() + }, + {}, + id="unicode-escape", + ), + pytest.param( + {"big.py": _DATA_MODULE}, + {(agent_type_check, "_LARGEST_UNREAD_SEARCHED"): LARGEST_PARSED_MODULE + 1}, + id="past-the-file-bound", + ), + pytest.param( + {"big.py": _DATA_MODULE}, + {(agent_type_check, "_UNREAD_SEARCH_TOTAL"): len(_DATA_MODULE) - 1}, + id="past-the-scan-total", + ), + pytest.param( + {"bundle.js": "const tool = 'computer-use-preview';\n"}, + {(agent_type_check, "_LARGEST_CODE_FILE"): 20}, + id="script-marker", + ), + # The packages a manifest lists aren't among the names searched for. + pytest.param( + {"package.json": json.dumps({"dependencies": {"puppeteer": "1"}})}, + {(agent_type_check, "_LARGEST_CODE_FILE"): 20}, + id="package-json", + ), + ], +) +def test_an_unread_file_that_may_name_a_tool_cuts(tmp_path, monkeypatch, files, totals): + for (module, name), value in totals.items(): + monkeypatch.setattr(module, name, value) + repo = _tree(tmp_path / "source", {"main.py": "import openai\n", **files}) + assert browser_scan(repo) == ([], frozenset({"file_size"})) + + +def test_past_the_search_total_a_file_isnt_opened(tmp_path, monkeypatch): + repo = _tree( + tmp_path / "source", + {"main.py": "import openai\n", "big.py": _DATA_MODULE, "big_too.py": _DATA_MODULE}, + ) + monkeypatch.setattr(agent_type_check, "_UNREAD_SEARCH_TOTAL", len(_DATA_MODULE) - 1) + opened: list[str] = [] + real = Path.open + + def spy(self, *args, **kwargs): + opened.append(self.name) + return real(self, *args, **kwargs) + + monkeypatch.setattr(Path, "open", spy) + assert browser_scan(repo) == ([], frozenset({"file_size"})) + # Neither fits what's left of the total, so neither is read to be searched. + assert "big.py" not in opened and "big_too.py" not in opened + + +@pytest.mark.parametrize( + "files, totals", + [ + # "computer" in prose starts no tool marker. + pytest.param( + {"big.py": _DATA_MODULE.replace("weather today", "computer", 1)}, + {}, + id="prose-computer", + ), + pytest.param( + {"big.py": b"# coding: latin-1\n" + _DATA_MODULE.encode()}, + {}, + id="raw-safe-coding", + ), + pytest.param( + {"bundle.js": "const answer = 42; // past twenty bytes\n"}, + {(agent_type_check, "_LARGEST_CODE_FILE"): 20}, + id="script", + ), + # The second file's search spends what's left exactly. + pytest.param( + {"big.py": _DATA_MODULE, "big_too.py": _DATA_MODULE}, + {(agent_type_check, "_UNREAD_SEARCH_TOTAL"): 2 * len(_DATA_MODULE)}, + id="at-the-scan-total", + ), + ], +) +def test_an_unread_file_that_names_no_tool_is_no_cut( + tmp_path, monkeypatch, files, totals +): + for (module, name), value in totals.items(): + monkeypatch.setattr(module, name, value) + repo = _tree(tmp_path / "source", {"main.py": "import openai\n", **files}) + assert browser_scan(repo) == ([], frozenset()) + + +def _unreadable(monkeypatch, repo: Path) -> None: + real = Path.open + + def refused(self, *args, **kwargs): + if self.name == "big.py": + raise PermissionError("refused") + return real(self, *args, **kwargs) + + monkeypatch.setattr(Path, "open", refused) + + +def _sized(size: int): + real = agent_type_check._file_size + return lambda path: size if path.name == "big.py" else real(path) + + +def _not_a_file(monkeypatch, repo: Path) -> None: + # Read, it would give nothing, and nothing names no tool. + (repo / "big.py").unlink() + (repo / "big.py").symlink_to(os.devnull) + monkeypatch.setattr( + agent_type_check, "_file_size", _sized(LARGEST_PARSED_MODULE + 1) + ) + + +def _grown(monkeypatch, repo: Path) -> None: + # Its size taken under the bound, it's past it when read. + (repo / "big.py").write_text("x = 1\n" * (5 * 1024 * 1024 // 6), encoding="utf-8") + monkeypatch.setattr( + agent_type_check, "_file_size", _sized(LARGEST_PARSED_MODULE + 1) + ) + + +@pytest.mark.parametrize("make", [_unreadable, _not_a_file, _grown]) +def test_an_unread_file_that_cant_be_cleared_cuts(tmp_path, monkeypatch, make): + repo = _tree( + tmp_path / "source", {"main.py": "import openai\n", "big.py": _DATA_MODULE} + ) + make(monkeypatch, repo) + assert browser_scan(repo) == ([], frozenset({"file_size"})) + + +def test_the_unread_search_is_charged_to_its_own_total(tmp_path, monkeypatch): + # Read first by name: charged to the scan's totals, it would leave the agent's file unread. + repo = _tree( + tmp_path / "source", + {"a_data.py": _DATA_MODULE, "z_agent.py": "import playwright\n"}, + ) + monkeypatch.setattr(agent_type_check, "_LARGEST_SCAN", 100) + monkeypatch.setattr(agent_type_check, "_LARGEST_PYTHON_SCAN", 100) + assert browser_scan(repo) == (["playwright"], frozenset()) + # The search's total is shared by the scan's files, in the order they're read. + agent_type_check._scan.cache_clear() + _tree(repo, {"b_data.py": _DATA_MODULE}) + monkeypatch.setattr(agent_type_check, "_UNREAD_SEARCH_TOTAL", len(_DATA_MODULE)) + assert browser_scan(repo) == (["playwright"], frozenset({"file_size"})) + + +@pytest.mark.parametrize( + "manifest, names", + [ + # A trailing comma: npm refuses it too. + pytest.param( + b'{"name": "x", "dependencies": {"react": "1"},}', [], id="trailing-comma" + ), + # Node reads a byte that isn't UTF-8 as a replacement character. + pytest.param( + b'{"description": "\xff", "dependencies": {"playwright": "1"}}', + ["playwright"], + id="not-utf-8-in-a-string", + ), + # Read that way, a stray byte before the object is no JSON either. + pytest.param( + b'\xff{"dependencies": {"playwright": "1"}}', [], id="not-utf-8-before-it" + ), + # JSON.parse reads an integer of any length; json refuses one past 4,300 digits. + pytest.param( + b'{"dependencies": {"puppeteer": "22"}, "n": ' + b"1" * 5000 + b"}", + ["puppeteer"], + id="a-5000-digit-integer", + ), + ], +) +def test_a_package_json_npm_reads_or_refuses_is_no_cut(tmp_path, manifest, names): + repo = _tree( + tmp_path / "b", {"main.py": "import openai\n", "package.json": manifest} + ) + assert browser_scan(repo) == (names, frozenset()) + + +@pytest.mark.parametrize( + "claimed, text, then, outcome", + [ + ("chat", _FOUND_CHAT.format(claimed="chat"), "chat", "saved"), + ("chat", _FOUND_CHAT.format(claimed="chat"), "browser", "refused"), + ("browser", _FOUND_BROWSER, "chat", "saved"), + ("browser", _FOUND_BROWSER, "browser", "refused"), + ], + ids=[ + "chat-then-chat", + "chat-then-browser", + "browser-then-chat", + "browser-then-browser", + ], +) +def test_cut_and_found_join_in_one_look(tmp_path, claimed, text, then, outcome): + repo = _cut_repo(tmp_path / "source", **{"agent.py": "import playwright\n"}) + assert browser_scan(repo) == (["playwright"], frozenset({"file_size"})) + assert ( + SecondLook.prompt(claimed, ["playwright"], cut=frozenset({"file_size"})) == text + ) + output = tmp_path / "out" + submit = _submitter(output, repo) + assert _text(submit(_reading(claimed))) == _look(text) + answer = submit(_reading(then, second_look_read=_READ)) + if outcome == "saved": + assert not answer.get("is_error"), answer + assert load(output).modality == then + else: + assert _refused_with(output) == _FOUND_MESSAGE + + +@pytest.mark.parametrize("then, outcome", [("browser", "refused"), ("chat", "saved")]) +def test_a_cut_browser_claim_drops_nothing_found(tmp_path, then, outcome): + output = tmp_path / "out" + submit = _submitter(output, _cut_repo(tmp_path / "source")) + first = _text(submit(_reading("browser"))) + assert first == _look(_CUT_BROWSER) + assert "was found" not in first + answer = submit(_reading(then, second_look_read=_READ)) + if outcome == "saved": + assert not answer.get("is_error"), answer + assert load(output).modality == "chat" + else: + assert _refused_with(output) == _READ_MESSAGE + + +@pytest.mark.parametrize( + "files, totals, said", + [ + pytest.param( + { + "big.py": _OVERSIZED, + "big_too.py": _OVERSIZED, + "agent.py": "import playwright\ndef (\n", + }, + {}, + "agent type scan cut by file_size 2, parse_failed 1", + id="two-limits", + ), + # A break leaves every file after it unread too, the one that broke it among them. + pytest.param( + {f"m{at}.py": "import playwright\n" for at in range(4)}, + {(agent_type_check, "_LARGEST_SCAN"): 30}, + "agent type scan cut by scan_total 3", + id="scan_total", + ), + pytest.param( + {f"m{at}.py": b"# coding: utf-7\nimport playwright\n" for at in range(2)}, + {(python_source, "LARGEST_ESCAPED_TOTAL"): 0}, + "agent type scan cut by decode_total 1", + id="decode_total", + ), + pytest.param({"agent.py": "import playwright\n"}, {}, None, id="uncut"), + ], +) +def test_the_cut_is_logged_once(tmp_path, monkeypatch, caplog, files, totals, said): + _tree(tmp_path, files) + for (module, name), value in totals.items(): + monkeypatch.setattr(module, name, value) + with caplog.at_level(logging.WARNING, logger=agent_type_check.__name__): + browser_scan(tmp_path) + browser_scan(tmp_path) + browser_tools_found(tmp_path) + first = [(record.levelno, record.getMessage()) for record in caplog.records] + caplog.clear() + agent_type_check._scan.cache_clear() + browser_scan(tmp_path) + again = [(record.levelno, record.getMessage()) for record in caplog.records] + assert first == again == ([] if said is None else [(logging.WARNING, said)]) + + +def _shared_repo(root: Path) -> Path: + return _tree( + root, + { + "agents/a/agent.py": "import common.web\n", + "agents/a/big.py": _OVERSIZED, + "common/__init__.py": "", + "common/web.py": "import playwright\n", + }, + ) + + +def test_one_call_walks_once(tmp_path, monkeypatch): + repo = _shared_repo(tmp_path / "source") + walks: list[str] = [] + scans: list[int] = [] + real_walk, real_scan = ( + agent_type_check.imported_by_agent, + agent_type_check._tools_in, + ) + + def walk(root, subfolder, **kwargs): + walks.append(subfolder) + return real_walk(root, subfolder, **kwargs) + + def scan(*args, **kwargs): + scans.append(1) + return real_scan(*args, **kwargs) + + monkeypatch.setattr(agent_type_check, "imported_by_agent", walk) + monkeypatch.setattr(agent_type_check, "_tools_in", scan) + reached: list[Path] = [] + assert browser_scan(repo, "agents/a", reached) == ( + ["playwright"], + frozenset({"file_size"}), + ) + assert walks == ["agents/a"] + assert len(scans) == 1 + assert repo.resolve() / "common" / "web.py" in reached + assert browser_tools_in_agent(repo, "agents/a") == ["playwright"] + + agent_type_check._scan.cache_clear() + scans.clear() + assert browser_scan(repo) == (["playwright"], frozenset({"file_size"})) + assert browser_scan(repo) == (["playwright"], frozenset({"file_size"})) + assert browser_tools_found(repo) == ["playwright"] + assert len(scans) == 1 + assert agent_type_check._scan.cache_info().currsize == 1 + + +def test_no_root_is_no_scan(): + assert browser_scan(None) == ([], frozenset()) + + +def test_the_tickets_padding_gets_a_look(tmp_path): + # Three 126 KB files naming playwright only in a comment spend the parse total first. + pad = "# playwright\n" + "# padding\n" * (126 * 1024 // 10) + repo = _tree( + tmp_path / "source", + { + **{f"a_pad_{at}.py": pad for at in range(3)}, + "z_agent.py": "import playwright\n", + }, + ) + assert browser_scan(repo) == ([], frozenset({"parse_total"})) + answer = _submitter(tmp_path / "out", repo)(_reading("chat")) + assert _text(answer) == _look(_CUT_CHAT.format(claimed="chat")) + + +def test_a_large_honest_repository_gets_a_look(tmp_path, monkeypatch): + # Twenty 4 KB modules read before the agent's main file, against a total of 64 KB standing in + # for the 128 MiB one a 118 MB repository runs out of. + module = "".join(f"def f{at}():\n return {at}\n" for at in range(200)) + repo = _tree( + tmp_path / "source", + { + **{f"lib/mod_{at:02}.py": module for at in range(20)}, + "service/main.py": "from playwright.sync_api import sync_playwright\n", + }, + ) + assert browser_scan(repo) == (["playwright"], frozenset()) + monkeypatch.setattr(agent_type_check, "_LARGEST_SCAN", 64 * 1024) + agent_type_check._scan.cache_clear() + assert browser_scan(repo) == ([], frozenset({"scan_total"})) + answer = _submitter(tmp_path / "out", repo, save_checks="thread")(_reading("chat")) + assert _text(answer) == _look(_CUT_CHAT.format(claimed="chat")) + + + +def test_a_large_honest_repository_gets_a_look_from_the_childs_scan(tmp_path): + import inspect + + # The child's scan stops at its limits; a stand-in says so, since a patch here can't reach it. + repo = _tree( + tmp_path / "source", + { + "service/main.py": "from playwright.sync_api import sync_playwright\n", + "save-stand-in.json": json.dumps({"scan": "cut"}), + }, + ) + server = contract_tools( + tmp_path / "out", scan_root=repo, save_checks="child", save_stand_in="_save_stand_ins" + ) + handler = next(one for one in server.tools if one.name == "submit_contract").handler + + answer = asyncio.run(handler(_reading("chat"))) + + assert _text(answer) == _look(_CUT_CHAT.format(claimed="chat")) + assert inspect.getclosurevars(handler).nonlocals["checker"].ran_in_child == 0 + +def test_the_walks_stop_is_no_cut(tmp_path, caplog): + # Over the walk's 512 KB parse total in modules that import, none of them naming a tool. + module = "import os\n" + "x = 1\n" * (100 * 1024 // 6) + repo = _tree( + tmp_path / "source", + { + **{f"agents/a/m_{at}.py": module for at in range(6)}, + "agents/a/z_main.py": "import common.util\n", + "common/__init__.py": "", + "common/util.py": "x = 1\n", + }, + ) + with caplog.at_level(logging.WARNING, logger=agent_type_check.__name__): + assert browser_scan(repo, "agents/a") == ([], frozenset()) + assert any( + "stopped at its limits" in record.getMessage() for record in caplog.records + ) + output = tmp_path / "out" + answer = _submitter(output, repo, "agents/a")(_reading("chat")) + assert not answer.get("is_error"), answer + assert load(output).modality == "chat" + + +def test_ci_runs_this_file(): + workflows = ( + Path(__file__).resolve().parents[2] / ".github" / "workflows" / "sdk-smoke.yml" + ) + assert "tests/harness/test_browser_scan_cut.py" in workflows.read_text( + encoding="utf-8" + ) diff --git a/tests/harness/test_contract_tools_subfolder.py b/tests/harness/test_contract_tools_subfolder.py index cb826a185..49dc7e46f 100644 --- a/tests/harness/test_contract_tools_subfolder.py +++ b/tests/harness/test_contract_tools_subfolder.py @@ -586,7 +586,9 @@ def test_the_check_runs_off_the_loop_and_a_refusal_meanwhile_wins(tmp_path, repo from fi.alk.harness.failure_file import FAILURE_FILE destination = tmp_path / "out" - server = contract_tools(destination, source_root=repo, scan_root=repo, subfolder="agents/a") + server = contract_tools( + destination, source_root=repo, scan_root=repo, subfolder="agents/a", save_checks="thread" + ) arguments = payload( runtime={"language": "python"}, hard_constraints=["be polite"], data_store={"kind": "none"} ) @@ -938,9 +940,15 @@ def test_loading_counts_mapping_keys(repo): bounded_yaml_load(text, [30]) -def _racing_server(tmp_path, repo): +def _racing_server(tmp_path, repo, save_checks=None): destination = tmp_path / "out" - server = contract_tools(destination, source_root=repo, scan_root=repo, subfolder="agents/a") + server = contract_tools( + destination, + source_root=repo, + scan_root=repo, + subfolder="agents/a", + save_checks=save_checks, + ) return destination, server.tools[0].handler @@ -960,7 +968,7 @@ def test_the_contract_sent_last_is_saved_when_an_earlier_one_finishes_later( from fi.alk.harness import tools - destination, submit = _racing_server(tmp_path, repo) + destination, submit = _racing_server(tmp_path, repo, "thread") second_checked = threading.Event() real = tools._checked_contract @@ -991,7 +999,7 @@ def test_a_refusal_during_the_check_wins_though_its_failure_file_is_not_written( from fi.alk.harness import tools - destination, submit = _racing_server(tmp_path, repo) + destination, submit = _racing_server(tmp_path, repo, "thread") checking, refused = threading.Event(), threading.Event() real = tools._checked_contract @@ -1051,7 +1059,15 @@ async def beat(): worst[0], last = max(worst[0], now - last), now beating = asyncio.create_task(beat()) - result = await submit(_plain("crafted")) + # The scan doesn't parse the crafted file; searched as bytes, it names no browser tool, so + # leaving it unread doesn't cut the scan, and the reading is saved without a look. + # A save made while the session's checks still run answers "not checked in time" (on a + # loaded host they can outlast the deadline), and the model writes again: so does this. + for _ in range(60): + result = await submit(_plain("crafted")) + if not _text(result).startswith("not checked in time"): + break + await asyncio.sleep(0.5) done.set() await beating return result, worst[0] @@ -1123,7 +1139,7 @@ def test_an_earlier_contract_is_saved_when_the_later_ones_write_fails(tmp_path, from fi.alk.harness import tools - destination, submit = _racing_server(tmp_path, repo) + destination, submit = _racing_server(tmp_path, repo, "thread") second_written = threading.Event() real_check, real_save = tools._checked_contract, tools._save_contract @@ -2023,7 +2039,9 @@ def test_the_save_walks_the_agents_imports_once(tmp_path, repo, monkeypatch): lambda root, subfolder, **limits: walks.append(subfolder) or real(root, subfolder, **limits), ) destination = tmp_path / "out" - server = contract_tools(destination, source_root=repo, scan_root=repo, subfolder="agents/a") + server = contract_tools( + destination, source_root=repo, scan_root=repo, subfolder="agents/a", save_checks="thread" + ) _submit(server, payload(runtime={"language": "python"}), destination) @@ -2114,3 +2132,323 @@ def test_a_path_past_twice_the_system_limit_is_refused_before_any_lookup(monkeyp with pytest.raises(ValueError): runtime_selection.charged_path(Path("/" + "../" * 3000 + "x")) assert looked == [] + + +# The save's checks in the save checker's child. Each row runs its calls on one event loop, which +# the child serves, and counts the saves sent to it. + +_STAND_IN = "_save_stand_ins" +_PAST_THE_DEADLINE = 3.0 + + +def _child_server(tmp_path, repo): + destination = tmp_path / "out" + server = contract_tools( + destination, + source_root=repo, + scan_root=repo, + subfolder="agents/a", + save_checks="child", + save_stand_in=_STAND_IN, + ) + handler = server.tools[0].handler + return destination, handler, _checker(handler) + + +def _checker(handler): + import inspect + + return inspect.getclosurevars(handler).nonlocals["checker"] + + +def _session(repo: Path, **said) -> None: + (repo / "save-stand-in.json").write_text(json.dumps(said)) + + +def _text(answer) -> str: + return answer["content"][0]["text"] + + +async def _until(path: Path, seconds: float = 10.0) -> None: + import time + + until = time.monotonic() + seconds + while not path.exists(): + assert time.monotonic() < until, f"{path.name} never appeared" + await asyncio.sleep(0.005) + + +@pytest.mark.parametrize("together", [True, False], ids=["one-message", "one-after-another"]) +def test_a_call_cut_after_a_refused_one_saves_nothing(tmp_path, repo, together): + destination, submit, checker = _child_server(tmp_path, repo) + + async def calls(): + if together: + # The refused call holds the child at tool evidence until the cut call is sent, so + # the order the two reach the save lock is fixed, not left to the loop's scheduling. + refused = asyncio.ensure_future(submit(_plain("tool_evidence:wait model:refuse"))) + await _until(repo / ".tool_evidence-running") + cut = asyncio.ensure_future(submit(_plain("contract:sleep"))) + await asyncio.sleep(0.05) + (repo / ".tool_evidence-go").write_text("") + return await refused, await cut + return await submit(_plain("model:refuse")), await submit(_plain("contract:sleep")) + + first, second = asyncio.run(calls()) + + assert "stand-in: model refused" in _text(first) + assert "not checked in time" in _text(second) + assert not (destination / "contract.json").exists() + assert checker.ran_in_child == 2 + + +def test_a_save_during_the_session_job_is_not_checked_in_time(tmp_path, repo): + _session(repo, scan={"after": ".go"}) + destination, submit, checker = _child_server(tmp_path, repo) + + async def calls(): + early = await submit(_plain("early")) + (repo / ".go").write_text("") + return early, await submit(_plain("later")) + + early, later = asyncio.run(calls()) + + assert "the folder's checks are still running" in _text(early) + assert "second-look" not in _text(early) + assert not later.get("is_error"), later + assert _saved(destination)["notes"] == "later" + assert checker.ran_in_child == 1 + + +def test_what_a_refusal_asked_holds_across_a_call_cut_after_it(tmp_path, repo): + destination, submit, checker = _child_server(tmp_path, repo) + + async def calls(): + return [ + await submit(_plain(notes)) + for notes in ("model:refuse", "model:refuse contract:sleep", "model:refuse") + ] + + refused, cut, kept = asyncio.run(calls()) + + assert "model: if the model is refused again, it is left out" in _text(refused) + assert "not checked in time" in _text(cut) + # Asked once, by the refusal: the citation is left out rather than refused again. + assert not kept.get("is_error"), kept + assert "model" not in _saved(destination) + assert checker.ran_in_child == 3 + + +def test_a_quote_given_in_one_call_checks_the_lines_another_names_in_the_same_message( + tmp_path, repo +): + (repo / "agents" / "a" / "prompt.py").write_text('PROMPT = "Answer in one short line."\n') + destination, submit, checker = _child_server(tmp_path, repo) + quote = "You are a careful assistant who confirms every order before adding an item to it." + lines = [{"path": "agents/a/prompt.py", "start_line": 1, "end_line": 1}] + + async def one_message(): + return await asyncio.gather( + submit({**_plain("quoted"), "system_prompt_excerpt": quote}), + submit({**_plain("named"), "system_prompt_from": lines}), + ) + + quoted, named = asyncio.run(one_message()) + + # The quote's call is asked for lines, and the lines' call is checked against that quote. + assert "system_prompt_from names no lines" in _text(quoted) + assert "don't hold the quote you gave before" in _text(named) + assert not (destination / "contract.json").exists() + assert checker.ran_in_child == 1 + + +def test_null_optional_fields_save_in_the_child_and_pass_the_quick_check(tmp_path, repo): + from fi.alk.harness import tools + + destination, submit, checker = _child_server(tmp_path, repo) + arguments = { + **_plain("nulls"), + "one_liner": None, + "agent_languages": None, + "runtime": {"language": "python", "command": None, "workdir": None}, + } + + assert tools._quick_schema(arguments, sorted(arguments)) is None + answer = asyncio.run(submit(arguments)) + + assert not answer.get("is_error"), answer + assert _saved(destination)["runtime"]["workdir"] == "agents/a" + assert checker.ran_in_child == 1 + + +def test_the_scan_wait_and_the_checks_share_one_deadline(tmp_path, repo): + import time + + from fi.alk.harness.save_checker import SAVE_DEADLINE + + _session(repo, scan={"after": ".go", "sleep": 1.0}) + destination, submit, checker = _child_server(tmp_path, repo) + + async def call(): + await _until(repo / ".scan-waiting") + (repo / ".go").write_text("") + started = time.monotonic() + # Under the deadline alone, but not in what the scan's wait leaves of it. + answer = await submit(_plain("model:sleep:0.8")) + return answer, time.monotonic() - started + + answer, took = asyncio.run(call()) + + assert "not checked in time: submit it once, alone" in _text(answer) + assert not (destination / "contract.json").exists() + assert took < SAVE_DEADLINE + 0.4 + assert checker.ran_in_child == 1 + + +def test_a_call_on_a_second_loop_runs_in_this_process(tmp_path, repo): + destination, submit, checker = _child_server(tmp_path, repo) + + first = asyncio.run(submit(_plain("first loop"))) + second = asyncio.run(submit(_plain("second loop"))) + + assert not first.get("is_error"), first + assert not second.get("is_error"), second + assert _saved(destination)["notes"] == "second loop" + assert checker.ran_in_child == 1 + + +def test_a_second_loop_call_reads_the_code_the_folder_reaches_itself(tmp_path, repo): + (repo / "agents" / "a" / "agent.py").write_text("from common import tracing\n") + (repo / "common").mkdir() + (repo / "common" / "__init__.py").write_text("") + (repo / "common" / "tracing.py").write_text("import traceai_openai\n") + _session(repo, scan="real", instrumentation={"hold": 10}) + destination, submit, checker = _child_server(tmp_path, repo) + + async def first_call(): + await _until(repo / ".scan-done") + return await submit(_plain("first loop")) + + first = asyncio.run(first_call()) + # The scan is cached from the child, which keeps what it reached: this loop walks it again. + second = asyncio.run(submit(_plain("second loop"))) + + assert "not checked in time: submit it once, alone" in _text(first) + assert not second.get("is_error"), second + assert _saved(destination)["instrumentation"] == "traceai" + assert checker.ran_in_child == 0 + + +def test_an_instrumentation_part_that_raises_twice_saves_with_none(tmp_path, repo): + from fi.alk.harness.save_checker import SessionChecksFailed + + (repo / "agents" / "a" / "agent.py").write_text("import traceai_openai\n") + _session(repo, instrumentation="raise") + destination, submit, checker = _child_server(tmp_path, repo) + + async def calls(): + with pytest.raises(SessionChecksFailed): + await submit(_plain("first")) + return await submit(_plain("second")) + + answer = asyncio.run(calls()) + + assert not answer.get("is_error"), answer + assert _saved(destination)["instrumentation"] == "none" + assert checker.ran_in_child == 1 + + +def test_after_two_deaths_the_checks_run_here_with_the_sessions_instrumentation(tmp_path, repo): + _session(repo, instrumentation={"value": "traceai"}) + destination, submit, checker = _child_server(tmp_path, repo) + + async def calls(): + ended = [await submit(_plain("tool_evidence:exit")) for _ in range(2)] + return ended, await submit(_plain("after")) + + ended, after = asyncio.run(calls()) + + assert all("not checked in time" in _text(one) for one in ended), ended + assert not after.get("is_error"), after + # This folder's own code says none: the answer is the one the child's session job gave. + assert _saved(destination)["instrumentation"] == "traceai" + assert checker.ran_in_child == 2 + + +def test_a_job_with_no_roots_starts_no_child(tmp_path): + import multiprocessing + + from fi.alk.harness import save_checker + + before = {child.pid for child in multiprocessing.active_children()} + server = contract_tools(tmp_path / "out", save_checks="child") + handler = server.tools[0].handler + checker = _checker(handler) + + answer = asyncio.run(handler(_plain("no roots"))) + + assert not answer.get("is_error"), answer + assert checker.ran_in_child == 0 + assert checker not in save_checker._OPEN + assert {child.pid for child in multiprocessing.active_children()} <= before + + +def test_a_save_sent_before_the_child_is_up_waits_for_it(tmp_path, repo): + destination, submit, checker = _child_server(tmp_path, repo) + + answer = asyncio.run(submit(_plain("at once"))) + + assert not answer.get("is_error"), answer + assert _saved(destination)["notes"] == "at once" + assert checker.ran_in_child == 1 + + +def test_a_refusal_during_the_childs_check_wins_though_its_failure_file_is_not_written( + tmp_path, repo, monkeypatch +): + from fi.alk.harness import agent_type_check, tools + + destination, submit, checker = _child_server(tmp_path, repo) + + def unwritable(*args): + raise OSError("disk full") + + # The reading's verdict, as SecondLook asks for it. + monkeypatch.setattr( + agent_type_check, + "decide", + lambda claimed, *args, **kwargs: "refuse" if claimed == "browser" else "keep", + ) + monkeypatch.setattr(tools, "write_failure_file", unwritable) + + async def both(): + saving = asyncio.ensure_future(submit(_plain("tool_evidence:wait"))) + # The refusal comes once the check is under way in the child. + await _until(repo / ".tool_evidence-running", 1.5) + with pytest.raises(OSError): + await submit({**_plain("refusing"), "modality": "browser"}) + (repo / ".tool_evidence-go").write_text("") + return await saving + + saving = asyncio.run(both()) + + assert not (destination / "contract.json").exists() + assert "Don't submit again" in _text(saving) + assert checker.ran_in_child == 1 + + +def test_the_childs_trial_passes_on_a_fork_server_the_agent_scan_started(tmp_path, repo): + import multiprocessing.forkserver + import time + + from fi.alk.harness.scan_folders import FolderChecker + + # The fork server's preload is fixed by whoever starts it first; here, the agent scan. + multiprocessing.forkserver._forkserver._stop() + with FolderChecker() as folders: + folders.check(repo, "agents/a", deadline=time.monotonic() + 30) + destination, submit, checker = _child_server(tmp_path, repo) + answer = asyncio.run(submit(_plain("after the agent scan"))) + + assert not answer.get("is_error"), answer + assert checker.ran_in_child == 1 diff --git a/tests/harness/test_save_checker.py b/tests/harness/test_save_checker.py new file mode 100644 index 000000000..87ac8f572 --- /dev/null +++ b/tests/harness/test_save_checker.py @@ -0,0 +1,1073 @@ +"""The save checker's parent side, against a fake child that speaks the pipe's messages. + +The fake runs through the same forkserver context as the real child, so kills, restarts, deaths +and the watchdog act on real processes. Each child takes its script from the plan by the order it +started in, and writes its pid and every request it got next to the test, for the asserts. +""" + +from __future__ import annotations + +import asyncio +import functools +import json +import logging +import os +import threading +import sys +import time +from collections.abc import Callable +from multiprocessing.connection import Connection +from pathlib import Path +from typing import Any + +import pytest + +from fi.alk.harness import save_checker +from fi.alk.harness.contract import AgentContract +from fi.alk.harness.save_checker import ( + STAGES, + NotInTime, + Refused, + Saved, + SaveChecker, + SaveChecksFailed, + ScanAnswer, + SessionChecksFailed, + StandInFailed, + default_mode, + rss_bytes, +) + +PAYLOAD = { + "agent": "trial", + "modality": "chat", + "real_use_cases": ["o"], + "hard_constraints": ["be polite"], + "tools": [{"name": "t0", "args": ["x"]}], +} +CONTRACT = AgentContract.model_validate(PAYLOAD).model_dump(mode="json") +REFUSAL = {"content": [{"type": "text", "text": "Not accepted: the child said so."}], "is_error": True} +SCAN_LATE = "not checked in time: the folder's checks are still running; submit it once, alone" +SAVE_LATE = "not checked in time: submit it once, alone" +CUT = ScanAnswer([], frozenset({"deadline"})) +FOUND = ScanAnswer(["playwright"], frozenset()) +# Long enough for a child to start on a loaded machine; the rows that test a deadline set theirs. +WAIT = 60.0 +MIB = 1024 * 1024 +FAKE_LOGGER = "tests.save_checker.fake" + + +def _stage(name: str, problems=(), asked=(), held=None) -> list[Any]: + return [ + "send", + { + "kind": "stage", + "stage": name, + "problems": list(problems), + "asked": list(asked), + "held": dict(held or {}), + }, + ] + + +SCAN = ["send", {"kind": "scan", "found": ["playwright"], "cut": []}] +INSTRUMENTATION = ["send", {"kind": "instrumentation", "value": "traceai"}] +DONE = ["send", {"kind": "done", "contract": CONTRACT, "response": None, "asked": [], "held": {}}] +GOOD_SAVE = [*(_stage(name) for name in STAGES), DONE] +FAILED_SCAN = [ + "send", + {"kind": "failed", "part": "scan", "error": "OSError", "message": "boom", "trace": "trace"}, +] +FAILED_INSTRUMENTATION = [ + "send", + { + "kind": "failed", + "part": "instrumentation", + "error": "RuntimeError", + "message": "boom", + "trace": "trace", + }, +] +FAILED_SAVE = [ + "send", + {"kind": "failed", "part": "save", "error": "KeyError", "message": "'x'", "trace": "trace"}, +] +NEXT_CHILD_LINE = [ + "send", + { + "kind": "log", + "name": FAKE_LOGGER, + "level": logging.WARNING, + "message": "the next child is up", + "trace": None, + }, +] +# Each kind's runs: the nth request of a kind takes run n, the last one repeating. +DEFAULTS: dict[str, list[list[list[Any]]]] = { + "trial": [[["send", {"kind": "trial", "ok": True, "detail": ""}]]], + "stand_in": [[["send", {"kind": "stand_in", "ok": True, "detail": ""}]]], + "session": [[SCAN, INSTRUMENTATION]], + "save": [GOOD_SAVE], +} + + +def _step(conn: Connection, rid: int | None, step: list[Any], kept: list[bytes]) -> None: + op, arg = step + if op == "sleep": + time.sleep(arg) + elif op == "send": + conn.send({"id": rid, **arg}) + elif op == "die": + os._exit(arg) + elif op == "grow": + # Written through, so every page is resident. + kept.append(b"x" * arg) + elif op == "logs": + until = time.monotonic() + arg + while time.monotonic() < until: + conn.send( + { + "id": None, + "kind": "log", + "name": FAKE_LOGGER, + "level": logging.WARNING, + "message": "flood", + "trace": None, + } + ) + time.sleep(0.002) + + +def _fake_child(plan: dict[str, Any], conn: Connection) -> None: + folder = Path(plan["folder"]) + index = 0 + while True: + try: + fd = os.open(folder / f"child-{index}", os.O_CREAT | os.O_EXCL | os.O_WRONLY) + break + except FileExistsError: + index += 1 + os.write(fd, str(os.getpid()).encode()) + os.close(fd) + scripts = plan["children"] + script = scripts[min(index, len(scripts) - 1)] + kept: list[bytes] = [] + for step in script.get("up", []): + _step(conn, None, step, kept) + conn.send({"id": None, "kind": "up", "pid": os.getpid()}) + counts: dict[str, int] = {} + while True: + try: + request = conn.recv() + except (EOFError, OSError): + return + kind = request["kind"] + with open(folder / f"requests-{index}", "a", encoding="utf-8") as log: + log.write(json.dumps(request) + "\n") + runs = script.get(kind) or DEFAULTS[kind] + run = runs[min(counts.get(kind, 0), len(runs) - 1)] + counts[kind] = counts.get(kind, 0) + 1 + for step in run: + _step(conn, request["id"], step, kept) + + +def _checker( + tmp_path: Path, *children: dict[str, Any], stand_in: str | None = None +) -> tuple[SaveChecker, Path]: + folder = tmp_path / "fake" + folder.mkdir() + checker = SaveChecker( + scan_root=tmp_path, source_root=tmp_path, subfolder=None, mode="child", stand_in=stand_in + ) + checker._target = functools.partial( + _fake_child, {"folder": str(folder), "children": list(children) or [{}]} + ) + return checker, folder + + +def _pids(folder: Path) -> list[int]: + pids = [] + index = 0 + while (folder / f"child-{index}").exists(): + text = (folder / f"child-{index}").read_text() + pids.append(int(text) if text else 0) + index += 1 + return pids + + +def _requests(folder: Path, index: int) -> list[dict[str, Any]]: + path = folder / f"requests-{index}" + if not path.exists(): + return [] + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + + +def _kinds(folder: Path, index: int) -> list[str]: + return [request["kind"] for request in _requests(folder, index)] + + +def _gone(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + return False + + +def _eventually(condition: Callable[[], bool], seconds: float = 20.0) -> bool: + until = time.monotonic() + seconds + while not condition(): + if time.monotonic() > until: + return False + time.sleep(0.02) + return True + + +async def _until(condition: Callable[[], bool], seconds: float = WAIT) -> None: + until = time.monotonic() + seconds + while not condition(): + assert time.monotonic() < until, "the checker's child never got there" + await asyncio.sleep(0.02) + + +async def _up(checker: SaveChecker) -> None: + checker.start() + await _until(lambda: checker._child is not None and checker._child.ready) + + +async def _idle(checker: SaveChecker) -> None: + """Waits for a ready child with nothing in it, as after a restart.""" + checker.start() + await _until( + lambda: checker._child is not None + and checker._child.ready + and checker._child.request is None + and checker._child.killed is None + and not checker._child.gone + ) + + +async def _scan(checker: SaveChecker, seconds: float = WAIT) -> ScanAnswer | NotInTime: + return await checker.scan(deadline=asyncio.get_running_loop().time() + seconds) + + +async def _save( + checker: SaveChecker, + seconds: float = WAIT, + asked: set[str] | None = None, + held: dict[str, str] | None = None, +) -> Saved | Refused | NotInTime: + return await checker.check( + dict(PAYLOAD), + require_setup_contract=False, + asked=set() if asked is None else asked, + held={} if held is None else held, + deadline=asyncio.get_running_loop().time() + seconds, + ) + + +def _text(result: Any) -> str: + return result.response["content"][0]["text"] + + +def _readers() -> int: + return sum( + 1 for thread in threading.enumerate() if thread.name == "save-checker-reader" and thread.is_alive() + ) + + +@pytest.fixture +def warnings(caplog: pytest.LogCaptureFixture) -> pytest.LogCaptureFixture: + caplog.set_level(logging.WARNING) + return caplog + + +@pytest.fixture +def thread_checks(monkeypatch: pytest.MonkeyPatch) -> list[dict[str, Any]]: + """Stands in for today's in-process checks, recording how thread mode called them.""" + from fi.alk.harness import tools + + calls: list[dict[str, Any]] = [] + + def checked(payload: dict[str, Any], **kwargs: Any) -> AgentContract: + calls.append(kwargs) + return AgentContract.model_validate(CONTRACT) + + def scanned(scan_root: Any, subfolder: Any, reached: Any = None) -> tuple[list[str], frozenset[str]]: + calls.append({"scan": scan_root}) + return ["in-thread"], frozenset() + + monkeypatch.setattr(tools, "_checked_contract", checked) + monkeypatch.setattr(tools, "_tools_in_agent_folder", scanned) + return calls + + +# -- the pipe's rules --------------------------------------------------------------------------- + + +def test_a_save_waits_for_both_parts_and_a_message_for_another_request_is_dropped( + tmp_path: Path, +) -> None: + stale_stage = ["send", {**_stage("contract", ["stale"], ["stale"])[1], "id": 10_000}] + checker, folder = _checker( + tmp_path, + { + "session": [ + [ + ["send", {"id": 10_000, "kind": "scan", "found": ["stale"], "cut": []}], + SCAN, + ["sleep", 0.5], + INSTRUMENTATION, + ] + ], + "save": [[stale_stage, *GOOD_SAVE]], + }, + ) + asked: set[str] = set() + + async def main() -> tuple[Any, Any]: + saved = await _save(checker, asked=asked) + return saved, await _scan(checker) + + saved, scanned = asyncio.run(main()) + + assert isinstance(saved, Saved) and saved.contract.agent == "trial" + assert scanned == FOUND + assert "stale" not in asked + assert _kinds(folder, 0) == ["trial", "session", "save"] + # Only known once the instrumentation part had answered. + assert _requests(folder, 0)[2]["instrumentation"] == "traceai" + assert checker.ran_in_child == 1 + + +def test_a_save_past_its_deadline_kills_the_child_and_a_new_one_starts( + tmp_path: Path, warnings: pytest.LogCaptureFixture +) -> None: + checker, folder = _checker(tmp_path, {"save": [[_stage("contract"), ["sleep", 60]]]}, {}) + + async def main() -> Any: + await _scan(checker) + return await _save(checker, seconds=0.5) + + result = asyncio.run(main()) + + assert isinstance(result, NotInTime) and _text(result) == SAVE_LATE + first = _pids(folder)[0] + assert _eventually(lambda: _gone(first)) + assert _eventually(lambda: checker._child is not None and checker._child.ready) + assert len(_pids(folder)) == 2 + # The trial passed and the cache is complete: the new child gets neither again. + assert _kinds(folder, 1) == [] + assert ( + "save checker: the save's deadline passed in tool_evidence, prompt_copy, model, runtime; " + "nothing saved" in warnings.text + ) + assert checker._deaths == 0 + + +def test_the_session_job_cut_at_its_bound_is_cached_and_not_run_again( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, warnings: pytest.LogCaptureFixture +) -> None: + monkeypatch.setattr(save_checker, "SESSION_BOUND", 1.0) + checker, folder = _checker(tmp_path, {"session": [[["sleep", 60]]]}, {}) + + async def main() -> tuple[Any, Any]: + return await _scan(checker), await _save(checker) + + scanned, saved = asyncio.run(main()) + + assert scanned == CUT + assert isinstance(saved, Saved) + assert _kinds(folder, 0) == ["trial", "session"] + assert _kinds(folder, 1) == ["save"] + assert _requests(folder, 1)[0]["instrumentation"] == "none" + # The restarted child never got `session`, so the save carries the parent's level. + assert _requests(folder, 1)[0]["log_level"] == logging.getLogger().level + assert ( + "save checker: the session's checks ran past 1 s; the scan and instrumentation counts as cut" + in warnings.text + ) + assert checker._deaths == 0 + + +def test_a_death_during_the_session_job_keeps_what_answered_and_counts( + tmp_path: Path, warnings: pytest.LogCaptureFixture +) -> None: + checker, folder = _checker(tmp_path, {"session": [[INSTRUMENTATION, ["die", 3]]]}, {}) + + async def main() -> tuple[Any, int, Any]: + scanned = await _scan(checker) + deaths = checker._deaths + return scanned, deaths, await _save(checker) + + scanned, deaths, saved = asyncio.run(main()) + + assert scanned == CUT + assert deaths == 1 + assert isinstance(saved, Saved) + assert _kinds(folder, 1) == ["save"] + assert _requests(folder, 1)[0]["instrumentation"] == "traceai" + assert "save checker: the child ended (exit code 3)" in warnings.text + assert "the scan counts as cut" in warnings.text + + +def test_two_failed_instrumentation_parts_then_saves_go_on_with_none( + tmp_path: Path, warnings: pytest.LogCaptureFixture +) -> None: + checker, folder = _checker(tmp_path, {"session": [[SCAN, FAILED_INSTRUMENTATION]]}) + + async def main() -> tuple[SessionChecksFailed, Any]: + with pytest.raises(SessionChecksFailed) as first: + await _save(checker) + return first.value, await _save(checker) + + first, second = asyncio.run(main()) + + assert (first.part, first.error, first.message) == ("instrumentation", "RuntimeError", "boom") + assert isinstance(second, Saved) + # Sent again to the same child after the first failure; cached as cut after the second. + assert _kinds(folder, 0) == ["trial", "session", "session", "save"] + assert _requests(folder, 0)[3]["instrumentation"] == "none" + assert len(_pids(folder)) == 1 + # The type alone at ERROR: the message and trace can quote the customer's paths. + assert "save checker: instrumentation failed in the child: RuntimeError\n" in warnings.text + assert "boom" not in warnings.text + assert "the session's instrumentation failed twice" in warnings.text + + +def test_two_calls_in_one_message_after_a_first_failure_send_one_session(tmp_path: Path) -> None: + checker, folder = _checker( + tmp_path, + {"session": [[SCAN, FAILED_INSTRUMENTATION], [["sleep", 0.3], SCAN, INSTRUMENTATION]]}, + ) + + async def main() -> list[Any]: + with pytest.raises(SessionChecksFailed): + await _save(checker) + return list(await asyncio.gather(_save(checker), _save(checker))) + + results = asyncio.run(main()) + + assert all(isinstance(result, Saved) for result in results) + assert _kinds(folder, 0) == ["trial", "session", "session", "save", "save"] + assert checker.ran_in_child == 2 + + +def test_a_failed_scan_is_sent_again_and_an_answer_after_a_cut_is_dropped(tmp_path: Path) -> None: + checker, folder = _checker( + tmp_path, {"session": [[FAILED_SCAN, INSTRUMENTATION], [FAILED_SCAN, SCAN]]} + ) + + async def main() -> tuple[SessionChecksFailed, Any, Any]: + with pytest.raises(SessionChecksFailed) as first: + await _scan(checker) + second = await _scan(checker) + # Time for the real answer that follows the second failure to arrive. + await asyncio.sleep(0.5) + return first.value, second, await _scan(checker) + + first, second, third = asyncio.run(main()) + + assert first.part == "scan" + assert second == third == CUT + assert _kinds(folder, 0) == ["trial", "session", "session"] + + +def test_two_deaths_move_the_checks_into_this_process_for_good( + tmp_path: Path, warnings: pytest.LogCaptureFixture, thread_checks: list[dict[str, Any]] +) -> None: + checker, folder = _checker(tmp_path, {"save": [[["die", 7]]]}, stand_in="tests.fake_stand_in") + + async def main() -> list[Any]: + return [await _save(checker) for _ in range(3)] + + first, second, third = asyncio.run(main()) + + assert isinstance(first, NotInTime) and isinstance(second, NotInTime) + assert isinstance(third, Saved) + # The restarted child's stand_in answer didn't reset the count. + assert _kinds(folder, 0) == ["trial", "stand_in", "session", "save"] + assert _kinds(folder, 1) == ["stand_in", "save"] + assert checker.ran_in_child == 2 + assert "save checker: running the checks in this process: the child ended 2 times in a row" in ( + warnings.text + ) + # Thread mode takes the cached instrumentation, and a cached scan's reach isn't passed. + assert thread_checks[-1]["instrumentation"] == "traceai" + assert thread_checks[-1]["reached"] is None + assert _eventually(lambda: all(_gone(pid) for pid in _pids(folder))) + + +def test_deadline_kills_in_a_row_stay_in_the_child( + tmp_path: Path, warnings: pytest.LogCaptureFixture +) -> None: + checker, folder = _checker(tmp_path, {"save": [[["sleep", 60]]]}) + + async def main() -> list[Any]: + await _scan(checker) + results = [] + for _ in range(3): + await _idle(checker) + results.append(await _save(checker, seconds=0.5)) + await _idle(checker) + return results + + results = asyncio.run(main()) + + assert all(isinstance(result, NotInTime) for result in results) + assert checker.ran_in_child == 3 + assert checker._deaths == 0 + assert len(_pids(folder)) == 4 + assert "running the checks in this process" not in warnings.text + assert "the child ended" not in warnings.text + + +def test_a_save_whose_child_dies_answers_with_the_problems_that_arrived(tmp_path: Path) -> None: + checker, folder = _checker( + tmp_path, + { + "save": [ + [ + _stage( + "contract", + ["missing-storage-decision: declare data_store.kind"], + ["model"], + {"prompt-quote": "q"}, + ), + ["die", 5], + ] + ] + }, + {}, + ) + asked, held = {"tools"}, {"other": "kept"} + + result = asyncio.run(_save(checker, asked=asked, held=held)) + + assert isinstance(result, Refused) + assert "missing-storage-decision" in _text(result) + assert asked == {"tools", "model"} + assert held == {"other": "kept", "prompt-quote": "q"} + assert checker._deaths == 1 + + +def test_a_restart_leaves_one_reader_thread(tmp_path: Path) -> None: + before = _readers() + checker, folder = _checker(tmp_path, {"save": [[["sleep", 60]]]}, {}) + + async def main() -> None: + await _scan(checker) + await _save(checker, seconds=0.3) + await _idle(checker) + + asyncio.run(main()) + + assert len(_pids(folder)) == 2 + assert _eventually(lambda: _readers() == before + 1) + + +def test_a_stage_exception_fails_the_call_and_is_not_a_death( + tmp_path: Path, warnings: pytest.LogCaptureFixture +) -> None: + checker, folder = _checker(tmp_path, {"save": [[_stage("contract"), FAILED_SAVE], GOOD_SAVE]}) + + async def main() -> tuple[SaveChecksFailed, Any]: + with pytest.raises(SaveChecksFailed) as failed: + await _save(checker) + return failed.value, await _save(checker) + + failed, saved = asyncio.run(main()) + + assert (failed.error, failed.message) == ("KeyError", "'x'") + assert isinstance(saved, Saved) + assert len(_pids(folder)) == 1 + assert checker._deaths == 0 + assert "save checker: save failed in the child: KeyError\n" in warnings.text + assert "'x'" not in warnings.text + + +def test_a_kill_while_the_child_logs_and_the_next_childs_lines_arrive( + tmp_path: Path, warnings: pytest.LogCaptureFixture +) -> None: + checker, folder = _checker( + tmp_path, {"save": [[["logs", 60]]]}, {"up": [NEXT_CHILD_LINE]} + ) + + async def main() -> tuple[Any, Any]: + await _scan(checker) + cut = await _save(checker, seconds=0.5) + return cut, await _save(checker) + + cut, saved = asyncio.run(main()) + + assert isinstance(cut, NotInTime) + assert isinstance(saved, Saved) + lines = [record.getMessage() for record in warnings.records if record.name == FAKE_LOGGER] + assert "flood" in lines + assert "the next child is up" in lines + + +def test_the_watchdog_kills_a_child_that_grows_past_its_cap( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, warnings: pytest.LogCaptureFixture +) -> None: + checker, folder = _checker(tmp_path, {"save": [[["grow", 300 * MIB], ["sleep", 60]]]}, {}) + + async def main() -> Any: + await _scan(checker) + resident = save_checker._rss(_pids(folder)[0]) + assert resident is not None + monkeypatch.setattr(save_checker, "WATCH_CAP", resident + 150 * MIB) + return await _save(checker) + + result = asyncio.run(main()) + + assert isinstance(result, NotInTime) + assert "save checker: the child ended (watchdog)" in warnings.text + assert checker._deaths == 1 + + +def test_the_cache_written_after_its_loop_closed_is_found_on_a_new_loop( + tmp_path: Path, warnings: pytest.LogCaptureFixture +) -> None: + checker, folder = _checker(tmp_path, {"session": [[["sleep", 1.0], SCAN, INSTRUMENTATION]]}) + + async def early() -> Any: + await _up(checker) + return await _scan(checker, seconds=0.2) + + first = asyncio.run(early()) + assert isinstance(first, NotInTime) and _text(first) == SCAN_LATE + assert _eventually(lambda: checker._scan is not None) + + # A scan in this process would find nothing in an empty folder, so this is the child's answer. + assert asyncio.run(_scan(checker)) == FOUND + assert "save checker: a call from a second event loop runs in this process" in warnings.text + + +@pytest.mark.parametrize( + ("trial", "why"), + [ + ( + [["send", {"kind": "trial", "ok": False, "detail": "the scan found []"}]], + "the trial failed: the scan found []", + ), + ([["sleep", 60]], "the trial did not answer within 0.5 s"), + ([["die", 4]], "the child ended before the trial answered"), + ], +) +def test_a_failed_or_late_trial_moves_the_checks_into_this_process( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + warnings: pytest.LogCaptureFixture, + thread_checks: list[dict[str, Any]], + trial: list[Any], + why: str, +) -> None: + monkeypatch.setattr(save_checker, "TRIAL_SECONDS", 0.5) + checker, folder = _checker(tmp_path, {"trial": [trial]}) + + async def main() -> tuple[Any, Any]: + return await _scan(checker), await _save(checker) + + scanned, saved = asyncio.run(main()) + + assert scanned == ScanAnswer(["in-thread"], frozenset()) + assert isinstance(saved, Saved) + assert checker.ran_in_child == 0 + assert f"save checker: running the checks in this process: {why}" in warnings.text + # A failed trial isn't a death. + assert checker._deaths == 0 + assert _eventually(lambda: all(_gone(pid) for pid in _pids(folder))) + + +def test_rss_bytes_reads_a_statm_line_and_a_ps_line() -> None: + assert rss_bytes("statm", "12345 2048 300 10 0 900 0\n", 4096) == 2048 * 4096 + assert rss_bytes("ps", " 51200\n", 4096) == 51200 * 1024 + with pytest.raises(ValueError): + rss_bytes("statm", "", 4096) + with pytest.raises(ValueError): + rss_bytes("ps", "RSS\n", 4096) + + +def test_ci_runs_this_file() -> None: + workflows = Path(__file__).resolve().parents[2] / ".github" / "workflows" / "sdk-smoke.yml" + assert "tests/harness/test_save_checker.py" in workflows.read_text(encoding="utf-8") + + +# -- what SaveChecker promises submit_contract ------------------------------------------------ + + +def test_a_scan_not_in_time_kills_nothing_and_the_session_job_runs_once(tmp_path: Path) -> None: + checker, folder = _checker(tmp_path, {"session": [[["sleep", 3.0], SCAN, INSTRUMENTATION]]}) + + async def main() -> list[Any]: + await _up(checker) + first = await _scan(checker, seconds=0.3) + second = await _scan(checker, seconds=0.3) + return [first, second, await _scan(checker)] + + first, second, third = asyncio.run(main()) + + assert isinstance(first, NotInTime) and _text(first) == SCAN_LATE + assert isinstance(second, NotInTime) + assert third == FOUND + assert len(_pids(folder)) == 1 + assert not _gone(_pids(folder)[0]) + assert _kinds(folder, 0) == ["trial", "session"] + + +def test_a_save_cut_in_any_stage_saves_nothing_and_changes_nothing(tmp_path: Path) -> None: + # The stages that answered set `asked` and `held` with no problem, as the prompt copy's + # confirmed quote does: a cut applies none of it. + children = [ + {"save": [[*(_stage(name, (), ["model"], {"prompt-quote-confirmed": "c"}) for name in STAGES[:at]), ["sleep", 60]]]} + for at in range(5) + ] + checker, folder = _checker(tmp_path, *children, {}) + asked, held = {"tools"}, {"prompt-quote": "kept"} + + async def main() -> list[Any]: + await _scan(checker) + results = [] + for _ in STAGES: + await _idle(checker) + results.append(await _save(checker, seconds=0.5, asked=asked, held=held)) + return results + + results = asyncio.run(main()) + + assert all(isinstance(result, NotInTime) for result in results), results + assert all(_text(result) == SAVE_LATE for result in results) + assert asked == {"tools"} + assert held == {"prompt-quote": "kept"} + + +@pytest.mark.parametrize("at", range(1, len(STAGES))) +def test_a_save_cut_after_a_problem_answers_that_problem(tmp_path: Path, at: int) -> None: + arrived = [ + _stage(STAGES[0], ["model-unconfirmed: the model isn't cited"], ["model"], {"prompt-quote": "q"}), + *(_stage(name, (), ["model"], {"prompt-quote": "q"}) for name in STAGES[1:at]), + ] + checker, folder = _checker(tmp_path, {"save": [[*arrived, ["sleep", 60]]]}, {}) + asked, held = {"tools"}, {"other": "kept"} + + async def main() -> Any: + await _scan(checker) + return await _save(checker, seconds=0.5, asked=asked, held=held) + + result = asyncio.run(main()) + + assert isinstance(result, Refused) + assert "model-unconfirmed" in _text(result) + assert asked == {"tools", "model"} + assert held == {"other": "kept", "prompt-quote": "q"} + + +def test_a_save_in_time_is_saved_or_refused_as_done_says(tmp_path: Path) -> None: + refused = [ + *(_stage(name, ["prompt-quote: not found"] if name == "prompt_copy" else ()) for name in STAGES), + [ + "send", + { + "kind": "done", + "contract": None, + "response": REFUSAL, + "asked": ["prompt-quote"], + "held": {"prompt-quote": "q2"}, + }, + ], + ] + saved_run = [ + *(_stage(name) for name in STAGES), + ["send", {**DONE[1], "asked": ["model"], "held": {"confirmed": "c"}}], + ] + checker, folder = _checker(tmp_path, {"save": [saved_run, refused]}) + asked, held = {"tools"}, {"prompt-quote": "q1"} + + async def main() -> tuple[Any, Any]: + first = await _save(checker, asked=asked, held=held) + snapshot = (set(asked), dict(held)) + return (first, snapshot), await _save(checker, asked=asked, held=held) + + (first, snapshot), second = asyncio.run(main()) + + assert isinstance(first, Saved) and first.contract == AgentContract.model_validate(CONTRACT) + assert snapshot == ({"tools", "model"}, {"prompt-quote": "q1", "confirmed": "c"}) + assert isinstance(second, Refused) and second.response == REFUSAL + assert asked == {"tools", "model", "prompt-quote"} + assert held == {"prompt-quote": "q2", "confirmed": "c"} + assert checker.ran_in_child == 2 + + +def test_a_call_on_a_second_loop_runs_in_this_process_and_kills_nothing( + tmp_path: Path, warnings: pytest.LogCaptureFixture, thread_checks: list[dict[str, Any]] +) -> None: + checker, folder = _checker(tmp_path, {}) + + first = asyncio.run(_save(checker)) + + async def elsewhere() -> list[Any]: + return [await _save(checker), await _save(checker)] + + later = asyncio.run(elsewhere()) + + assert isinstance(first, Saved) + assert all(isinstance(result, Saved) for result in later) + assert checker.ran_in_child == 1 + assert [call["instrumentation"] for call in thread_checks] == ["traceai", "traceai"] + assert warnings.text.count("a call from a second event loop runs in this process") == 1 + assert len(_pids(folder)) == 1 and not _gone(_pids(folder)[0]) + + +def test_a_cancel_while_waiting_kills_nothing(tmp_path: Path) -> None: + checker, folder = _checker( + tmp_path, + { + "session": [[["sleep", 1.0], SCAN, INSTRUMENTATION]], + "save": [[["sleep", 1.0], *GOOD_SAVE]], + }, + ) + + async def main() -> Any: + await _up(checker) + # Cancelled while it waits on the session job. + waiting = asyncio.ensure_future(_save(checker)) + await asyncio.sleep(0.3) + waiting.cancel() + with pytest.raises(asyncio.CancelledError): + await waiting + # Cancelled while it waits on the lock behind a save in the child. + first = asyncio.ensure_future(_save(checker)) + await _until(lambda: checker.ran_in_child == 1) + behind = asyncio.ensure_future(_save(checker)) + await asyncio.sleep(0.2) + behind.cancel() + with pytest.raises(asyncio.CancelledError): + await behind + return await first + + result = asyncio.run(main()) + + assert isinstance(result, Saved) + assert len(_pids(folder)) == 1 and not _gone(_pids(folder)[0]) + assert checker.ran_in_child == 1 + + +def test_a_cancel_while_its_own_request_is_in_the_child_kills_and_restarts_it( + tmp_path: Path, warnings: pytest.LogCaptureFixture +) -> None: + checker, folder = _checker(tmp_path, {"save": [[["sleep", 60]]]}, {}) + + async def main() -> Any: + await _scan(checker) + task = asyncio.ensure_future(_save(checker)) + await _until(lambda: checker.ran_in_child == 1) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + return await _save(checker) + + result = asyncio.run(main()) + + assert isinstance(result, Saved) + first, second = _pids(folder) + assert _eventually(lambda: _gone(first)) + assert not _gone(second) + assert checker._deaths == 0 + assert "the child ended" not in warnings.text + + +def test_no_child_outlives_close(tmp_path: Path) -> None: + before = _readers() + checker, folder = _checker(tmp_path, {}) + + assert asyncio.run(_scan(checker)) == FOUND + checker.close() + checker.close() + + assert _eventually(lambda: all(_gone(pid) for pid in _pids(folder))) + assert _readers() == before + + +def test_close_during_start_stops_what_start_started(tmp_path: Path) -> None: + checker, folder = _checker(tmp_path, {}) + + checker.start() + began = time.monotonic() + checker.close() + + assert time.monotonic() - began < save_checker.JOIN_SECONDS + 1.0 + assert checker._start_thread is not None and not checker._start_thread.is_alive() + assert _eventually(lambda: all(_gone(pid) for pid in checker.pids)) + + +def test_a_closed_checker_answers_in_this_process( + tmp_path: Path, warnings: pytest.LogCaptureFixture, thread_checks: list[dict[str, Any]] +) -> None: + checker, folder = _checker(tmp_path, {}) + assert asyncio.run(_scan(checker)) == FOUND + checker.close() + + async def main() -> tuple[Any, Any]: + return await _scan(checker), await _save(checker) + + scanned, saved = asyncio.run(main()) + + assert scanned == FOUND + assert isinstance(saved, Saved) + assert checker.ran_in_child == 0 + assert thread_checks and thread_checks[-1]["instrumentation"] == "traceai" + assert warnings.text.count("save checker: closed; running the checks in this process") == 1 + assert len(_pids(folder)) == 1 + + +def test_a_stand_in_that_fails_raises_and_never_falls_back( + tmp_path: Path, warnings: pytest.LogCaptureFixture +) -> None: + checker, folder = _checker( + tmp_path, + {"stand_in": [[["send", {"kind": "stand_in", "ok": False, "detail": "ModuleNotFoundError: x"}]]]}, + stand_in="tests.missing_stand_in", + ) + + async def main() -> None: + with pytest.raises(StandInFailed, match="ModuleNotFoundError"): + await _scan(checker) + with pytest.raises(StandInFailed): + await _save(checker) + + asyncio.run(main()) + + # The trial checks the real functions before any stand-in is put in place. + assert _kinds(folder, 0) == ["trial", "stand_in"] + assert checker.ran_in_child == 0 + assert "running the checks in this process" not in warnings.text + assert _eventually(lambda: all(_gone(pid) for pid in _pids(folder))) + + +def test_a_job_with_no_roots_starts_no_child() -> None: + checker = SaveChecker(scan_root=None, source_root=None, subfolder=None, mode="child") + checker.start() + + assert checker.pids == [] and checker._start_thread is None + + +def test_the_default_mode_follows_the_switch(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("ALK_SAVE_CHECKS", "thread") + assert default_mode() == "thread" + monkeypatch.delenv("ALK_SAVE_CHECKS") + assert default_mode() == "child" + + +def test_a_child_never_up_after_a_passed_trial_is_a_death_and_replaced( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, warnings: pytest.LogCaptureFixture +) -> None: + from fi.alk.harness import save_checker + + monkeypatch.setattr(save_checker, "START_SECONDS", 0.5) + checker, folder = _checker(tmp_path, {"save": [[["sleep", 60]]]}, {"up": [["sleep", 60]]}, {}) + + async def main() -> tuple[Any, Any]: + await _scan(checker) + cut = await _save(checker, seconds=0.5) + return cut, await _save(checker, seconds=10) + + cut, after = asyncio.run(main()) + + # Not replaced, the session would answer "not checked in time" until it closed. + assert isinstance(cut, NotInTime) + assert isinstance(after, Saved), after + assert len(_pids(folder)) == 3 + assert "save checker: the child ended (start)" in warnings.text + + +def test_a_save_whose_deadline_passed_on_the_lock_is_not_sent( + tmp_path: Path, warnings: pytest.LogCaptureFixture +) -> None: + checker, folder = _checker(tmp_path, {"save": [[*(_stage(name) for name in STAGES), DONE]]}) + + async def main() -> tuple[Any, Any]: + await _scan(checker) + await _idle(checker) + late = await checker.check( + {"agent": "x"}, + require_setup_contract=False, + asked=set(), + held={}, + deadline=asyncio.get_running_loop().time() - 0.001, + ) + # The lock was given back: the next save in time is sent and saved. + return late, await _save(checker) + + late, after = asyncio.run(main()) + + assert isinstance(late, NotInTime) + assert isinstance(after, Saved), after + assert checker.ran_in_child == 1 + assert len(_pids(folder)) == 1 + + +@pytest.mark.skipif(sys.version_info < (3, 11), reason="3.10 keeps wait_for, which can swallow it") +def test_a_cancel_in_the_same_tick_as_the_answer_still_cancels() -> None: + from fi.alk.harness import save_checker + + async def main() -> None: + loop = asyncio.get_running_loop() + answer: asyncio.Future[str] = loop.create_future() + waiting = asyncio.ensure_future(save_checker._by(answer, loop.time() + 5)) + await asyncio.sleep(0) + + def both() -> None: + answer.set_result("done") + waiting.cancel() + + loop.call_soon(both) + with pytest.raises(asyncio.CancelledError): + await waiting + + asyncio.run(main()) + + +def test_a_session_answer_after_an_idle_death_does_not_reset_the_death_count( + tmp_path: Path, warnings: pytest.LogCaptureFixture, thread_checks: list[dict[str, Any]] +) -> None: + # A part fails, the idle child then dies, and the next child answers the session again + # before it dies too: only a save's answer resets the count, so that is two in a row. + checker, folder = _checker( + tmp_path, + {"session": [[SCAN, FAILED_INSTRUMENTATION, ["sleep", 0.5], ["die", 3]]]}, + {"save": [[["die", 7]]]}, + ) + + async def main() -> tuple[Any, Any]: + # No call waits on the failed part, so nothing sends the session again before the death. + await _up(checker) + await _until(lambda: checker._failures["instrumentation"] == 1 and checker._deaths == 1) + cut = await _save(checker) + return cut, await _save(checker) + + cut, after = asyncio.run(main()) + + assert isinstance(cut, NotInTime) + assert isinstance(after, Saved) + assert checker.ran_in_child == 1 + assert "running the checks in this process: the child ended 2 times in a row" in warnings.text + + +def test_a_call_waiting_when_the_checker_closes_runs_in_this_process( + tmp_path: Path, thread_checks: list[dict[str, Any]] +) -> None: + checker, folder = _checker(tmp_path, {"session": [[["sleep", 5.0], SCAN, INSTRUMENTATION]]}) + + async def main() -> Any: + await _up(checker) + waiting = asyncio.ensure_future(_save(checker, seconds=4)) + await asyncio.sleep(0.3) + checker.close() + return await waiting + + result = asyncio.run(main()) + + # Not left to wait out its deadline: a closed checker runs the call here. + assert isinstance(result, Saved) + assert checker.ran_in_child == 0 diff --git a/tests/harness/test_save_child.py b/tests/harness/test_save_child.py new file mode 100644 index 000000000..a918ec8d1 --- /dev/null +++ b/tests/harness/test_save_child.py @@ -0,0 +1,718 @@ +"""The save checks' child, driven in this process over a real pipe. + +``serve`` runs on a thread here rather than in a forked child, so each row can patch what the +child runs and read every message it sends, in the order it sent them. +""" + +from __future__ import annotations + +import ast +import contextlib +import logging +import multiprocessing +import os +import sys +import tempfile +import threading +from pathlib import Path +from typing import Any + +import pytest + +from fi.alk.harness import ( + agent_type_check, + forkserver_preload, + instrumentation_evidence, + save_child, + scan_folders, + tools, +) +from fi.alk.harness.contract import AgentContract + +_WAIT = 30.0 +_TERMINAL = {"trial", "stand_in", "done"} + + +@contextlib.contextmanager +def _child(*, install_logging: bool = False): + parent, child = multiprocessing.Pipe() + thread = threading.Thread( + target=save_child.serve, + args=(child,), + kwargs={"install_logging": install_logging}, + daemon=True, + ) + thread.start() + try: + yield parent + finally: + # The child reads end of file and returns. + parent.close() + thread.join(10) + child.close() + assert not thread.is_alive() + + +def _read(parent, *, count: int | None = None, until=None) -> list[dict[str, Any]]: + """Messages, in order, until ``until(message)`` is true or ``count`` arrived.""" + got: list[dict[str, Any]] = [] + while True: + assert parent.poll(_WAIT), f"the child stopped answering after {got}" + message = parent.recv() + got.append(message) + if until is not None and until(message): + return got + if count is not None and len(got) == count: + return got + + +def _up(parent) -> dict[str, Any]: + return _read(parent, count=1)[0] + + +def _answered(rid: int): + return lambda message: message["id"] == rid and message["kind"] in _TERMINAL | {"failed"} + + +_SAME = object() + + +def _session(parent, rid: int, root: Path | None, subfolder: str | None, *, scan_root=_SAME): + """Sends a session request and reads until both parts have answered.""" + source = str(root) if root is not None else None + parent.send( + { + "id": rid, + "kind": "session", + "scan_root": source if scan_root is _SAME else scan_root, + "source_root": source, + "subfolder": subfolder, + "log_level": logging.WARNING, + } + ) + got: list[dict[str, Any]] = [] + parts = 0 + while parts < 2: + message = _read(parent, count=1)[0] + got.append(message) + if message["kind"] in {"scan", "instrumentation"} or ( + message["kind"] == "failed" and message["part"] in {"scan", "instrumentation"} + ): + parts += 1 + return got + + +def _save_request(rid: int, root: Path, payload: dict[str, Any], **changes: Any): + request = { + "id": rid, + "kind": "save", + "payload": payload, + "source_root": str(root), + "subfolder": save_child.TRIAL_SUBFOLDER, + "require_setup_contract": False, + "instrumentation": "traceai", + "asked": [], + "held": {}, + } + request.update(changes) + return request + + +def _write(root: Path, tree: dict[str, str]) -> Path: + for name, text in tree.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return root + + +def _kinds(messages: list[dict[str, Any]]) -> list[str]: + return [m["kind"] for m in messages if m["kind"] != "log"] + + +@pytest.fixture +def trial_dir(tmp_path, monkeypatch): + """The trial's temporary folder lands here, so a row can see that it is gone.""" + made = tmp_path / "temp" + made.mkdir() + monkeypatch.setattr(tempfile, "tempdir", str(made)) + return made + + +# P0, P1 + + +def test_up_comes_first_with_the_childs_pid(): + with _child() as parent: + assert _up(parent) == {"id": None, "kind": "up", "pid": os.getpid()} + + +def test_two_requests_are_answered_in_the_order_they_came(): + with _child() as parent: + _up(parent) + parent.send({"id": 1, "kind": "stand_in", "module": "no_such_stand_in_module"}) + parent.send( + { + "id": 2, + "kind": "session", + "scan_root": None, + "source_root": None, + "subfolder": None, + "log_level": logging.WARNING, + } + ) + got = _read(parent, count=3) + assert [(m["id"], m["kind"]) for m in got] == [ + (1, "stand_in"), + (2, "instrumentation"), + (2, "scan"), + ] + assert got[1]["value"] == "none" + assert got[2] == {"id": 2, "kind": "scan", "found": [], "cut": []} + + +def test_the_loop_returns_when_the_parents_end_closes(): + parent, child = multiprocessing.Pipe() + thread = threading.Thread(target=save_child.serve, args=(child,), kwargs={"install_logging": False}) + thread.start() + _up(parent) + parent.close() + thread.join(10) + child.close() + assert not thread.is_alive() + + +# P2 + + +def test_the_trial_passes_on_the_real_checks_and_sends_only_its_answer(trial_dir): + with _child() as parent: + _up(parent) + parent.send({"id": 7, "kind": "trial"}) + got = _read(parent, until=_answered(7)) + assert [m for m in got if m["kind"] != "log"] == [ + {"id": 7, "kind": "trial", "ok": True, "detail": ""} + ] + assert not list(trial_dir.glob("alk-trial-*")) + + +def test_the_trial_fails_when_the_scan_finds_nothing(trial_dir, monkeypatch): + monkeypatch.setattr(tools, "browser_scan", lambda *args, **kwargs: ([], frozenset())) + with _child() as parent: + _up(parent) + parent.send({"id": 7, "kind": "trial"}) + got = _read(parent, until=_answered(7)) + answer = got[-1] + assert answer["kind"] == "trial" and answer["ok"] is False + assert "scan found []" in answer["detail"] + assert _kinds(got) == ["trial"] + assert not list(trial_dir.glob("alk-trial-*")) + + +def test_the_trial_fails_when_instrumentation_ignores_what_the_scan_reached( + trial_dir, monkeypatch +): + real = instrumentation_evidence.detect_instrumentation + monkeypatch.setattr( + instrumentation_evidence, + "detect_instrumentation", + lambda source_root, reached=(): real(source_root), + ) + with _child() as parent: + _up(parent) + parent.send({"id": 7, "kind": "trial"}) + answer = _read(parent, until=_answered(7))[-1] + assert answer["ok"] is False + assert "'none', not 'traceai'" in answer["detail"] + + +def test_the_trial_fails_when_a_stage_goes_unreported(trial_dir, monkeypatch): + real = tools._checked_contract + + def skips_the_model(*args, progress, **kwargs): + def report(stage, problems): + if stage != "model": + progress(stage, problems) + + return real(*args, progress=report, **kwargs) + + monkeypatch.setattr(tools, "_checked_contract", skips_the_model) + with _child() as parent: + _up(parent) + parent.send({"id": 7, "kind": "trial"}) + answer = _read(parent, until=_answered(7))[-1] + assert answer["ok"] is False + assert "stages" in answer["detail"] + + +def test_the_trial_fails_when_the_runtime_stage_is_missing(trial_dir, monkeypatch): + def answers_early(payload, *, progress, **kwargs): + for stage in ["contract", "tool_evidence", "prompt_copy", "model"]: + progress(stage, []) + return _trial_contract() + + monkeypatch.setattr(tools, "_checked_contract", answers_early) + with _child() as parent: + _up(parent) + parent.send({"id": 7, "kind": "trial"}) + answer = _read(parent, until=_answered(7))[-1] + assert answer["ok"] is False + assert "stages" in answer["detail"] + + +def test_the_trial_fails_when_the_save_refuses_its_contract(trial_dir, monkeypatch): + def refuses(payload, *, progress, **kwargs): + for stage in save_child._STAGE_ORDER: + progress(stage, []) + return {"content": [], "is_error": True} + + monkeypatch.setattr(tools, "_checked_contract", refuses) + with _child() as parent: + _up(parent) + parent.send({"id": 7, "kind": "trial"}) + answer = _read(parent, until=_answered(7))[-1] + assert answer == { + "id": 7, + "kind": "trial", + "ok": False, + "detail": "the save refused the trial's contract", + } + + +def test_an_exception_in_the_trial_fails_it_without_a_failed_message(trial_dir, monkeypatch): + def no_room(*args, **kwargs): + raise OSError("no room for the trial's folder") + + monkeypatch.setattr(tempfile, "mkdtemp", no_room) + with _child() as parent: + _up(parent) + parent.send({"id": 7, "kind": "trial"}) + got = _read(parent, until=_answered(7)) + assert _kinds(got) == ["trial"] + assert got[-1]["ok"] is False + assert got[-1]["detail"] == "OSError: no room for the trial's folder" + + +def test_a_trial_that_raises_after_writing_its_tree_still_removes_it(trial_dir, monkeypatch): + real = Path.write_text + written: list[Path] = [] + + def second_fails(self, *args, **kwargs): + if written: + raise OSError("the disk filled") + written.append(self) + return real(self, *args, **kwargs) + + monkeypatch.setattr(Path, "write_text", second_fails) + with _child() as parent: + _up(parent) + parent.send({"id": 7, "kind": "trial"}) + got = _read(parent, until=_answered(7)) + assert got[-1]["ok"] is False and written + assert not list(trial_dir.glob("alk-trial-*")) + + +# P2a + + +@pytest.fixture +def stand_in_module(tmp_path, monkeypatch): + """Writes a stand-in module importable by name, and puts back whatever it replaced.""" + monkeypatch.syspath_prepend(str(tmp_path)) + for attribute in ("_copy_prompt", "model_problems"): + monkeypatch.setattr(tools, attribute, getattr(tools, attribute)) + names: list[str] = [] + + def write(name: str, text: str) -> str: + (tmp_path / f"{name}.py").write_text(text, encoding="utf-8") + names.append(name) + return name + + yield write + for name in names: + sys.modules.pop(name, None) + + +def test_a_stand_in_that_loads_replaces_the_checks_it_names(stand_in_module): + name = stand_in_module( + "save_child_stand_in_ok", + "def copy_prompt(contract, source_root):\n" + " return ['stood in']\n" + "STAGES = {('tools', '_copy_prompt'): copy_prompt,\n" + " ('fi.alk.harness.tools', 'model_problems'): copy_prompt}\n", + ) + with _child() as parent: + _up(parent) + parent.send({"id": 3, "kind": "stand_in", "module": name}) + answer = _read(parent, until=_answered(3))[-1] + assert answer == {"id": 3, "kind": "stand_in", "ok": True, "detail": ""} + stood_in = sys.modules[name].copy_prompt + assert tools._copy_prompt is stood_in + assert tools.model_problems is stood_in + + +def test_a_stand_in_that_does_not_load_says_why(): + with _child() as parent: + _up(parent) + parent.send({"id": 3, "kind": "stand_in", "module": "no_such_stand_in_module"}) + answer = _read(parent, until=_answered(3))[-1] + assert answer["ok"] is False + assert answer["detail"].startswith("ModuleNotFoundError: ") + + +def test_a_stand_in_naming_a_missing_check_changes_nothing(stand_in_module): + before = tools._copy_prompt + name = stand_in_module( + "save_child_stand_in_typo", + "def stand(*args):\n" + " return []\n" + "STAGES = {('tools', '_copy_prompt'): stand, ('tools', '_copy_promt'): stand}\n", + ) + with _child() as parent: + _up(parent) + parent.send({"id": 3, "kind": "stand_in", "module": name}) + answer = _read(parent, until=_answered(3))[-1] + assert answer["ok"] is False + assert "_copy_promt" in answer["detail"] + assert tools._copy_prompt is before + + +def test_a_stand_in_cannot_replace_a_module_outside_the_kit(stand_in_module, monkeypatch): + monkeypatch.setattr(os, "getpid", os.getpid) + name = stand_in_module( + "save_child_stand_in_outside", + "STAGES = {('os', 'getpid'): lambda: 0}\n", + ) + with _child() as parent: + _up(parent) + parent.send({"id": 3, "kind": "stand_in", "module": name}) + answer = _read(parent, until=_answered(3))[-1] + assert answer["ok"] is False + assert os.getpid() != 0 + + +# P3 + + +_REACHING_TREE = { + "agent/main.py": "from lib import trace\nimport playwright\n", + "lib/__init__.py": "", + "lib/trace.py": "import traceai_openai\n", +} + + +@pytest.fixture +def calls(monkeypatch): + """Records each check the session job calls, in order, through the real one.""" + seen: list[tuple[str, Any]] = [] + scan = tools._tools_in_agent_folder + detect = instrumentation_evidence.detect_instrumentation + walk = agent_type_check.code_the_agent_reaches + + def scanning(scan_root, subfolder, reached=None): + seen.append(("scan", subfolder)) + return scan(scan_root, subfolder, reached) + + def detecting(source_root, reached=()): + reached = list(reached) + seen.append(("instrumentation", reached)) + return detect(source_root, reached) + + def walking(source_root, subfolder): + seen.append(("walk", subfolder)) + return walk(source_root, subfolder) + + monkeypatch.setattr(tools, "_tools_in_agent_folder", scanning) + monkeypatch.setattr(instrumentation_evidence, "detect_instrumentation", detecting) + monkeypatch.setattr(agent_type_check, "code_the_agent_reaches", walking) + return seen + + +def test_with_a_folder_the_scan_runs_first_and_instrumentation_reads_its_reach(tmp_path, calls): + root = _write(tmp_path, _REACHING_TREE) + with _child() as parent: + _up(parent) + got = _session(parent, 4, root, "agent") + assert _kinds(got) == ["scan", "instrumentation"] + assert got[0] == {"id": 4, "kind": "scan", "found": ["playwright"], "cut": []} + assert got[1] == {"id": 4, "kind": "instrumentation", "value": "traceai"} + assert [name for name, _ in calls] == ["scan", "instrumentation"] + assert calls[1][1], "instrumentation read the scan's reach" + + +def test_without_a_folder_instrumentation_runs_first(tmp_path, calls): + root = _write(tmp_path, {"main.py": "import traceai_openai\nimport playwright\n"}) + with _child() as parent: + _up(parent) + got = _session(parent, 4, root, None) + assert _kinds(got) == ["instrumentation", "scan"] + assert got[0]["value"] == "traceai" + assert got[1]["found"] == ["playwright"] + assert [name for name, _ in calls] == ["instrumentation", "scan"] + + +def test_a_scan_of_another_root_has_instrumentation_walk_the_agents_reach(tmp_path, calls): + root = _write(tmp_path, _REACHING_TREE) + with _child() as parent: + _up(parent) + got = _session(parent, 4, root, "agent", scan_root=None) + assert got[1]["value"] == "traceai" + assert [name for name, _ in calls] == ["scan", "walk", "instrumentation"] + + +def test_a_missing_folder_gives_none(tmp_path, calls): + root = _write(tmp_path, _REACHING_TREE) + with _child() as parent: + _up(parent) + got = _session(parent, 4, root, "nowhere") + assert got == [ + {"id": 4, "kind": "scan", "found": [], "cut": []}, + {"id": 4, "kind": "instrumentation", "value": "none"}, + ] + + +def test_a_scan_that_raised_has_instrumentation_walk_afresh(tmp_path, calls, monkeypatch): + root = _write(tmp_path, _REACHING_TREE) + partial = tmp_path / "half-read.py" + + def breaks(scan_root, subfolder, reached=None): + calls.append(("scan", subfolder)) + reached.append(partial) + raise RuntimeError("the scan broke") + + monkeypatch.setattr(tools, "_tools_in_agent_folder", breaks) + with _child() as parent: + _up(parent) + got = _session(parent, 4, root, "agent") + failed, traced = got + assert failed["kind"] == "failed" and failed["part"] == "scan" + assert (failed["error"], failed["message"]) == ("RuntimeError", "the scan broke") + assert "RuntimeError: the scan broke" in failed["trace"] + assert traced == {"id": 4, "kind": "instrumentation", "value": "traceai"} + assert [name for name, _ in calls] == ["scan", "walk", "instrumentation"] + assert partial not in calls[-1][1] + + +# P4, P6 + + +def test_a_part_that_raises_is_sent_as_failed_and_the_other_part_and_the_loop_go_on( + tmp_path, monkeypatch +): + root = _write(tmp_path, {"main.py": "import playwright\n"}) + + def raises(*args, **kwargs): + raise ValueError("instrumentation broke") + + monkeypatch.setattr(instrumentation_evidence, "detect_instrumentation", raises) + with _child() as parent: + _up(parent) + got = _session(parent, 5, root, None) + failed, scan = got + assert failed["kind"] == "failed" and failed["part"] == "instrumentation" + assert failed["error"] == "ValueError" + assert "ValueError: instrumentation broke" in failed["trace"] + assert scan["kind"] == "scan" and scan["found"] == ["playwright"] + parent.send({"id": 6, "kind": "stand_in", "module": "no_such_stand_in_module"}) + assert _read(parent, until=_answered(6))[-1]["kind"] == "stand_in" + + +def test_a_save_that_raises_is_sent_as_failed_and_the_loop_goes_on(tmp_path, monkeypatch): + def raises(*args, **kwargs): + raise KeyError("save broke") + + monkeypatch.setattr(tools, "_checked_contract", raises) + with _child() as parent: + _up(parent) + parent.send(_save_request(8, tmp_path, dict(save_child.TRIAL_PAYLOAD))) + got = _read(parent, until=_answered(8)) + assert _kinds(got) == ["failed"] + assert got[-1]["part"] == "save" and got[-1]["error"] == "KeyError" + parent.send({"id": 9, "kind": "stand_in", "module": "no_such_stand_in_module"}) + assert _read(parent, until=_answered(9))[-1]["kind"] == "stand_in" + + +# P5 + + +def _trial_contract() -> AgentContract: + return AgentContract.model_validate( + tools._without_nulls(tools.unwrapped(save_child.TRIAL_PAYLOAD)) + ) + + +def test_the_save_sends_each_stage_then_one_done_with_what_it_set(tmp_path, monkeypatch): + given: dict[str, Any] = {} + + def checks(payload, *, progress, asked, held, **kwargs): + given.update(kwargs, payload=payload) + progress("contract", ["first"]) + asked.add("model") + held["prompt-quote"] = "kept from before" + held["prompt-quote-confirmed"] = "a new quote" + for stage in ["tool_evidence", "prompt_copy", "model", "runtime"]: + progress(stage, []) + return _trial_contract() + + monkeypatch.setattr(tools, "_checked_contract", checks) + with _child() as parent: + _up(parent) + parent.send( + _save_request( + 8, + tmp_path, + dict(save_child.TRIAL_PAYLOAD), + asked=["prompt-quote"], + held={"prompt-quote": "kept from before", "another call's": "left alone"}, + instrumentation="none", + ) + ) + got = _read(parent, until=_answered(8)) + assert _kinds(got) == ["stage"] * 5 + ["done"] + assert [m["stage"] for m in got[:5]] == save_child._STAGE_ORDER + assert got[0] == { + "id": 8, + "kind": "stage", + "stage": "contract", + "problems": ["first"], + "asked": ["prompt-quote"], + "held": {}, + } + written = {"prompt-quote": "kept from before", "prompt-quote-confirmed": "a new quote"} + assert got[1]["asked"] == ["model", "prompt-quote"] + assert got[1]["held"] == written + done = got[-1] + assert done == { + "id": 8, + "kind": "done", + "contract": _trial_contract().model_dump(mode="json"), + "response": None, + "asked": ["model", "prompt-quote"], + "held": written, + } + assert given["source_root"] == tmp_path + assert given["subfolder"] == "agent" + assert given["require_setup_contract"] is False + assert given["instrumentation"] == "none" + assert given["payload"] == save_child.TRIAL_PAYLOAD + + +def test_a_refused_save_sends_its_response_in_done(tmp_path, monkeypatch): + response = {"content": [{"type": "text", "text": "Not accepted."}], "is_error": True} + + def refuses(payload, *, progress, **kwargs): + for stage in save_child._STAGE_ORDER: + progress(stage, ["no"] if stage == "prompt_copy" else []) + return response + + monkeypatch.setattr(tools, "_checked_contract", refuses) + with _child() as parent: + _up(parent) + parent.send(_save_request(8, tmp_path, dict(save_child.TRIAL_PAYLOAD))) + got = _read(parent, until=_answered(8)) + assert _kinds(got) == ["stage"] * 5 + ["done"] + assert got[2]["problems"] == ["no"] + assert got[-1]["contract"] is None + assert got[-1]["response"] == response + + +def test_the_real_checks_report_every_stage_for_an_accepted_and_a_refused_payload(tmp_path): + root = _write(tmp_path, save_child.TRIAL_TREE) + refused = dict( + save_child.TRIAL_PAYLOAD, + system_prompt_from=[{"path": "agent/absent.py", "start_line": 1, "end_line": 3}], + ) + with _child() as parent: + _up(parent) + parent.send(_save_request(8, root, dict(save_child.TRIAL_PAYLOAD))) + accepted = _read(parent, until=_answered(8)) + parent.send(_save_request(9, root, refused, held={"another call's": "left alone"})) + not_accepted = _read(parent, until=_answered(9)) + for got in (accepted, not_accepted): + assert _kinds(got) == ["stage"] * 5 + ["done"] + assert [m["stage"] for m in got[:5]] == save_child._STAGE_ORDER + assert accepted[-1]["contract"]["instrumentation"] == "traceai" + assert accepted[-1]["response"] is None + assert not_accepted[-1]["contract"] is None + assert not_accepted[-1]["response"]["is_error"] is True + assert not_accepted[2]["problems"], "the prompt's lines are missing" + assert "another call's" not in not_accepted[-1]["held"] + + +# P7 + + +def test_a_log_line_with_a_trace_reaches_the_parent_as_text_and_logging_is_put_back( + tmp_path, monkeypatch +): + root = logging.getLogger() + handlers, level = list(root.handlers), root.level + + def logs(payload, *, progress, **kwargs): + # The child's own logger masks, which keeps the trace only as text. + try: + raise ValueError("a fault worth a trace") + except ValueError: + save_child.logger.exception("the check fell over") + save_child.logger.debug("below the parent's level") + return {"content": [], "is_error": True} + + monkeypatch.setattr(tools, "_checked_contract", logs) + with _child(install_logging=True) as parent: + _up(parent) + assert [type(h) for h in root.handlers] == [save_child.PipeLogHandler] + parent.send( + { + "id": 1, + "kind": "session", + "scan_root": None, + "source_root": None, + "subfolder": None, + "log_level": logging.INFO, + } + ) + _read(parent, until=lambda message: message["kind"] == "scan") + assert root.level == logging.INFO + parent.send(_save_request(2, tmp_path, dict(save_child.TRIAL_PAYLOAD))) + got = _read(parent, until=_answered(2)) + logs_sent = [m for m in got if m["kind"] == "log" and m["name"] == save_child.logger.name] + assert len(logs_sent) == 1 + line = logs_sent[0] + assert (line["id"], line["level"]) == (None, logging.ERROR) + assert line["message"] == "the check fell over" + assert line["trace"].startswith("Traceback") + assert "ValueError: a fault worth a trace" in line["trace"] + assert root.handlers == handlers + assert root.level == level + + +def test_a_log_line_on_a_closed_pipe_is_dropped(): + parent, child = multiprocessing.Pipe() + child.close() + parent.close() + handler = save_child.PipeLogHandler(save_child._Pipe(child)) + record = logging.LogRecord("fi.alk.harness.tools", logging.WARNING, __file__, 1, "x", None, None) + handler.handle(record) + + +# The shared preload + + +def test_both_checkers_preload_one_list_that_names_the_save_checks(): + assert scan_folders._PRELOAD is forkserver_preload.PRELOAD + for name in ( + "fi.alk.harness.framework_detection", + "fi.alk.harness.tools", + "fi.alk.harness.runtime_repair", + "fi.alk.harness.save_child", + ): + assert name in forkserver_preload.PRELOAD + source = Path(forkserver_preload.__file__).read_text(encoding="utf-8") + assert not [ + node for node in ast.walk(ast.parse(source)) if isinstance(node, (ast.Import, ast.ImportFrom)) + ] + + +def test_ci_runs_this_file(): + workflows = Path(__file__).resolve().parents[2] / ".github" / "workflows" / "sdk-smoke.yml" + assert "tests/harness/test_save_child.py" in workflows.read_text(encoding="utf-8")