diff --git a/src/fi/alk/harness/scan_ai.py b/src/fi/alk/harness/scan_ai.py index d28b938f2..bdd22d9b2 100644 --- a/src/fi/alk/harness/scan_ai.py +++ b/src/fi/alk/harness/scan_ai.py @@ -168,7 +168,8 @@ def _events(self, received: Any, turn: Turn) -> list[Any]: self._on_turn_end() except Exception: # A failed cost note must not turn a paid answer into a retried one. The - # trace is logged (secret values masked), so a fault in the cost writer can be found. + # trace is logged, with only the platform's secret values masked, so a fault + # in the cost writer can be found. logger.warning("find_agents: the cost note after the turn failed", exc_info=True) @@ -412,7 +413,8 @@ async def _answered(work: Callable[..., str], *args: Any) -> dict[str, Any]: except asyncio.TimeoutError: return _error(_TOO_SLOW) except Exception: - # The model is told only that the call failed; the trace is logged, secret values masked. + # The model is told only that the call failed. The trace is logged, with only the + # platform's secret values masked. logger.warning("find_agents: a tool call failed", exc_info=True) return _error(_FAILED) diff --git a/src/fi/alk/harness/scan_document.py b/src/fi/alk/harness/scan_document.py index 7c38318b1..da44b05fa 100644 --- a/src/fi/alk/harness/scan_document.py +++ b/src/fi/alk/harness/scan_document.py @@ -21,7 +21,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any -from .scan_walk import fits_path_rule +from .scan_walk import ENV_NAMES_KEPT, fits_path_rule if TYPE_CHECKING: from .job import HarnessJob @@ -33,7 +33,6 @@ SCHEMA_VERSION = "futureagi.agent-scan.v1" MAX_AGENTS = 50 MAX_BYTES = 1_048_576 -ENV_NAMES_KEPT = 50 NAME_MAX = 255 EVIDENCE_MAX = 200 EVIDENCE_KEPT = 5 diff --git a/src/fi/alk/harness/scan_folders.py b/src/fi/alk/harness/scan_folders.py index 3fb460e7f..06d7193b4 100644 --- a/src/fi/alk/harness/scan_folders.py +++ b/src/fi/alk/harness/scan_folders.py @@ -67,7 +67,8 @@ def _answer(source: Path, subfolder: str) -> dict[str, Any]: return {"name": "unknown", "version": None} except Exception: # A fault in the check itself. The child stays up for the next folder, and sends the - # trace so the parent logs it masked and marks the row unfinished. + # trace so the parent logs it, with only the platform's secret values masked, and marks + # the row unfinished. return {_FAILED: traceback.format_exc()} diff --git a/src/fi/alk/harness/scan_walk.py b/src/fi/alk/harness/scan_walk.py index 3077c7b61..1c3741dad 100644 --- a/src/fi/alk/harness/scan_walk.py +++ b/src/fi/alk/harness/scan_walk.py @@ -24,9 +24,10 @@ import time import tokenize import warnings +from bisect import bisect_left from dataclasses import dataclass from pathlib import Path -from typing import Callable, Iterator, Literal +from typing import Callable, Iterable, Iterator, Literal from .framework_detection import ( _ALWAYS_SKIPPED, @@ -56,10 +57,17 @@ # A hundredth of a folder check's allowance: the crafted files that use a whole one # take seconds, so one file now costs about a second at most. _FILE_WORK = 1_000_000_000 -# What the walk holds grows with the names each file imports, not its bytes: about 33 KB a -# file for files that each import 200 names (measured 10 Nov, 1,389 MB peak at 40,000 such -# files), which leaves room for the folder check's child in a 4,096 MB sandbox. -_MAX_FILES = 40_000 +# What the walk and its join keep grows with the names the files hold, not with their bytes or +# their number: a crafted 51 KB file of 200 imports of unique dotted names keeps 671 KB, so +# 5,100 of them, inside the byte cap, kept 3.3 GiB (measured 10 Nov). So everything kept is +# charged, where it is kept, at `_block`'s estimate of the allocator's blocks, and the walk +# stops past this many bytes: about 1.9 GB for the scan's process with its baseline, which the +# folder check's child, started after the walk, shares in a 4,096 MB sandbox. +_MAX_HELD = 1_610_612_736 +# The files the join works on: Python files that import something or carry their own sign. The +# join's time grows with them; files that do neither don't count, so a repository of many +# small files isn't cut by their number. +_MAX_JOINED = 40_000 _MAX_BYTES = 268_435_456 _NAMES_PER_FILE = 200 _IMPORT_DEPTH = 10 @@ -71,9 +79,14 @@ # The prompt count is an estimate, and every file walked is listed, read or not: past this # many a tree of prompt files costs memory and adds nothing a person would act on. _MAX_PROMPT_PATHS = 20_000 -# How many folder entries one look for a build's hazard visits. The hazard is advice, so a -# tree too big to look through gives none rather than the walk's time. +# How many folder entries one look for a build's hazard visits, and how many bytes of subfolder +# paths it lists (what `os.walk` holds for the folders it has yet to enter, counted without +# refunds since its stack can't be seen). The hazard is advice, so a tree too big to look +# through gives none rather than the walk's time or memory. _HAZARD_ENTRIES = 200_000 +_HAZARD_PATH_BYTES = 67_108_864 +# The most setting names a row keeps: all the answer file writes. +ENV_NAMES_KEPT = 50 # No file system the scan runs on has a path this long, so no file can match. _MODULE_PATH_MAX = 4_096 @@ -199,6 +212,144 @@ class _Unread(Exception): pass +# CPython 3.12's sizes on a 64-bit build, as asked of the allocator: a str (`_text`), a tuple +# (`_TUPLE` and a slot per item), a set's own object, one dict or set entry with its share of +# the table, a facts object, and the join's share for a file that could be an agent (its sign, +# its component bookkeeping, its row) beside three path strings `_project_of` and +# `_subfolder` make. Each is then charged at the block the allocator gives it (`_block`). +_TUPLE = 40 +_SLOT = 8 +_SET = 216 +_ENTRY = 112 +_FACTS = 512 +_JOIN_SHARE = 768 +# What the join holds for every kept file however it's joined: its place in the sorted list of +# paths and in one search's set of files seen. +_JOIN_PER_FILE = 64 +# What the join holds whatever was kept: its result, and up to `_MAX_AGENTS` rows with their +# names and evidence. Charged once, when the walk starts. A build hazard's own look is +# bounded on its own (`_HAZARD_PATH_BYTES`). +_JOIN_BASE = 262_144 +# An int above 256 is an object of its own. +_SMALL_INT = 256 +_INT = 48 + + +def _block(size: int) -> int: + """The block the allocator gives ``size`` bytes: pymalloc's 16-byte classes up to 512, + then glibc's malloc, whose header and alignment this rounds above (the sandbox is Linux).""" + if size <= 512: + return (size + 15) // 16 * 16 + return (size + 16 + 15) // 16 * 16 + + +def _text(name: str) -> int: + """A str's block: 49 + one byte a character when ASCII, 80 + four bytes otherwise.""" + return _block(49 + len(name) if name.isascii() else 80 + 4 * len(name)) + + +def _tuple(items: int) -> int: + return _block(_TUPLE + _SLOT * items) + + +def _held_names(names: Iterable[str]) -> int: + """Names kept in a set or a dict, each with its entry.""" + return sum(_ENTRY + _text(name) for name in names) + + +def _joined(path: str, facts: _Facts) -> bool: + """Whether the file counts against `_MAX_JOINED`: it imports something or carries its own + sign. A graph target isn't known yet when its facts are kept, so it doesn't count.""" + return bool(facts.imports) or _sign(path, facts, {}) is not None + + +def _join_share(path: str) -> int: + return _JOIN_SHARE + 3 * _text(path) + + +def _held_for_join(path: str, facts: _Facts) -> int: + """What the join will hold for one kept file: its import list, its place in the sorted + paths and one search's seen set, and, for a file with its own sign, its share.""" + size = _ENTRY + _tuple(len(facts.imports)) + _JOIN_PER_FILE + if _sign(path, facts, {}) is not None: + size += _join_share(path) + return size + + +def _held_facts(path: str, facts: _Facts) -> int: + """At least what keeping one Python file's facts holds, in the walk and in the join after + it: the facts and every name in them, the path and its two dict entries (the facts and the + canonical key), and what the join will hold for it. No name is taken as shared, so it is + high where Python shares one.""" + size = _FACTS + _text(path) + 2 * _ENTRY + _held_for_join(path, facts) + size += _tuple(len(facts.imports)) + for level, parts in facts.imports: + size += ( + _block(_TUPLE + 2 * _SLOT) + + _tuple(len(parts)) + + sum(_text(part) for part in parts) + ) + if level > _SMALL_INT: + size += _INT + size += _tuple(len(facts.env)) + sum(_text(name) for name in facts.env) + for pairs in (facts.builds, facts.models, facts.clients): + size += _tuple(len(pairs)) + for name, line in pairs: + size += ( + _block(_TUPLE + 2 * _SLOT) + + _text(name) + + (_INT if line > _SMALL_INT else 0) + ) + return size + + +class _CapReached(Exception): + """A reading reached one of its caps and stops here. It is never one file's problem, so it + isn't in `_FILE_ERRORS`, and every caller catches it before those and before its own + `except Exception`.""" + + +class _CappedReader: + """The one capped reader behind the walk and the description. It counts reads, and the + bytes read, against its caps, and, with ``held``, what is kept (`charge`); what a cap means + is the caller's: the walk stops, and the description marks its row partial.""" + + def __init__( + self, *, files: int | None, total_bytes: int, held: int | None = None + ) -> None: + self._files = files + self._total_bytes = total_bytes + self._held = held + self.files_read = 0 + self.bytes_read = 0 + self.held = 0 + + def charge(self, size: int) -> None: + """Count ``size`` bytes about to be kept. Past the cap, `_CapReached`, and the caller + keeps none of it.""" + if self._held is None: + return + self.held += size + if self.held > self._held: + raise _CapReached() + + def read(self, path: str) -> bytes: + """The file's bytes, in the order every stop point relies on: the read count is checked + before reading (`_CapReached`); the system's own error for a file that can't be read + passes out uncounted; the read is counted; past the bytes cap, `_CapReached`, and the + file isn't used; and a file over the size one file is read at raises `_Unread`.""" + if self._files is not None and self.files_read >= self._files: + raise _CapReached() + data = _read_file(path, _FILE_BYTES + 1) + self.files_read += 1 + self.bytes_read += len(data) + if self.bytes_read > self._total_bytes: + raise _CapReached() + if len(data) > _FILE_BYTES: + raise _Unread() + return data + + class _ResultOutOfTime(Exception): """The step after the walk (joining files into agents) passed its deadline.""" @@ -620,12 +771,17 @@ def _script_names(data: bytes) -> set[str]: class _Projects: - """Which folders hold a project file, looked up once per folder.""" + """Which folders hold a project file, and each folder's nearest project, looked up once per + folder. ``charge`` counts each entry before it is kept.""" - def __init__(self, top: str, names: tuple[str, ...]) -> None: + def __init__( + self, top: str, names: tuple[str, ...], charge: Callable[[int], None] | None = None + ) -> None: self._top = top self._names = names + self._charge = charge self._file: dict[str, str | None] = {} + self._nearest: dict[str, str | None] = {} def file_in(self, folder: str) -> str | None: if folder not in self._file: @@ -635,16 +791,30 @@ def file_in(self, folder: str) -> str | None: if os.path.isfile(path) and not os.path.islink(path): found = name break + if self._charge is not None: + self._charge(_ENTRY + _text(folder)) self._file[folder] = found return self._file[folder] def nearest(self, folder: str) -> str | None: - while True: - if self.file_in(folder) is not None: - return folder - if not folder: - return None - folder = posixpath.dirname(folder) + """The nearest folder at or above ``folder`` with a project file. The first look at a + folder looks at every ancestor not yet looked at, so each is looked at once and later + looks follow the kept links.""" + if folder in self._nearest: + return self._nearest[folder] + chain = [] + current: str | None = folder + while current is not None and current not in self._nearest: + chain.append(current) + current = posixpath.dirname(current) if current else None + above = self._nearest[current] if current is not None else None + for here in reversed(chain): + if self.file_in(here) is not None: + above = here + if self._charge is not None: + self._charge(_ENTRY) + self._nearest[here] = above + return above def _sign(path: str, facts: _Facts, graph_files: dict[str, str]) -> Sign | None: @@ -734,12 +904,18 @@ def _components(nodes: list[str], edges: dict[str, list[str]]) -> list[list[str] def _source_files( - top: str, start: str, *, side: bool | None = None + top: str, + start: str, + *, + side: bool | None = None, + charge: Callable[[int], None] | None = None, ) -> Iterator[tuple[str, str, str]]: """Every file below ``start`` the scan may read, as (directory, folder relative to ``top``, name), in sorted order so that a walk stopped by a limit stops at the same file. Installed trees and the folders a build skips are never entered. ``side`` False gives only - files outside test, example and other side folders; True gives only the files in them.""" + files outside test, example and other side folders; True gives only the files in them. + ``charge`` counts each folder kept to enter, at its full path, which is what `os.walk` + holds until it enters it.""" for directory, directories, files in os.walk(start, followlinks=False): directories.sort() files.sort() @@ -751,6 +927,10 @@ def _source_files( directories[:] = _folders_to_enter(directory, directories, at_top=not relative) if side is False: directories[:] = [name for name in directories if not _is_side_part(name)] + if charge is not None and directories: + charge( + sum(2 * _SLOT + _text(os.path.join(directory, name)) for name in directories) + ) folder = "/".join(relative) for name in files: if side is None or _read_with_side_files(_join(folder, name)) is side: @@ -771,17 +951,21 @@ class _ShadowedNames: """Each project's own module names, read once per project, so an import of the customer's own ``agents`` package isn't read as a framework.""" - def __init__(self, top: str, projects: _Projects) -> None: + def __init__( + self, top: str, projects: _Projects, charge: Callable[[int], None] | None = None + ) -> None: self._top = top self._projects = projects + self._charge = charge self._by_project: dict[str, frozenset[str]] = {} def for_folder(self, folder: str) -> frozenset[str]: project = self._projects.nearest(folder) or "" if project not in self._by_project: - self._by_project[project] = _own_modules( - Path(self._top, project) if project else Path(self._top), set() - ) + names = _own_modules(Path(self._top, project) if project else Path(self._top), set()) + if self._charge is not None: + self._charge(_ENTRY + _block(_SET) + _held_names(names)) + self._by_project[project] = names return self._by_project[project] @@ -810,27 +994,40 @@ def __init__( self.faults = 0 # When the step after the walk must give up: see _RESULT_GRACE. self.result_deadline = deadline + _RESULT_GRACE - self.files_read = 0 - self.bytes_read = 0 + # Everything the walk and its join keep is charged here before it is kept (`_MAX_HELD`). + self.reader = _CappedReader(files=None, total_bytes=_MAX_BYTES, held=_MAX_HELD) + self.reader.charge(_JOIN_BASE) + # Files the join works on (`_MAX_JOINED`). + self.joined = 0 self.stopped: Literal["", "limit", "clock", "memory"] = "" self.source_outside_side = False self.script_outside_side = False - self.projects = _Projects(self.top, _PROJECT_FILES) - self.shadowed = _ShadowedNames(self.top, self.projects) - self._roots: dict[str, list[str]] = {} + self.projects = _Projects(self.top, _PROJECT_FILES, self.reader.charge) + self.shadowed = _ShadowedNames(self.top, self.projects, self.reader.charge) + # Each kept path, so a resolved import holds the walk's own string, not a copy. + self._keys: dict[str, str] = {} self._edges: dict[str, tuple[str, ...]] = {} + # For the rows: the kept paths in order, and each row folder's names and tools. + self._sorted: list[str] | None = None + self._owned_by: dict[str, tuple[list[str], int]] = {} def run(self) -> None: # The customer's own code first, then side folders, so tests and examples can't use - # up the file cap before the agent is reached. - for side in (False, True): - for directory, folder, name in _source_files(self.top, self.top, side=side): - if self.clock() > self.deadline: - self.stopped = "clock" - return - self._take(directory, folder, name) - if self.stopped: - return + # up the caps before the agent is reached. + try: + for side in (False, True): + for directory, folder, name in _source_files( + self.top, self.top, side=side, charge=self.reader.charge + ): + if self.clock() > self.deadline: + self.stopped = "clock" + return + self._take(directory, folder, name) + if self.stopped: + return + except _CapReached: + # A charge made outside one file's read: a folder's listing or a prompt path. + self.stopped = "limit" def _take(self, directory: str, folder: str, name: str) -> None: path = _join(folder, name) @@ -844,53 +1041,67 @@ def _take(self, directory: str, folder: str, name: str) -> None: and len(self.prompt_paths) < _MAX_PROMPT_PATHS and not os.path.islink(full) ): + self.reader.charge(2 * _SLOT + _text(path)) self.prompt_paths.append(path) if not (name.endswith(".py") or name == _LANGGRAPH or name in _TEMPLATES): return # Every read of one file sits in this one `try`: a file the tokenizer, the JSON - # reader or the decoder rejects is that file's problem, never the walk's. + # reader or the decoder rejects is that file's problem, never the walk's. What is kept + # is charged first, so at a cap the walk stops with none of the file kept. try: - data = self._read(full) - if data is None: - return + data = self.reader.read(full) if name.endswith(".py"): - self.facts[path] = _python_facts(data, self.shadowed.for_folder(folder)) + self._keep_facts(path, _python_facts(data, self.shadowed.for_folder(folder))) elif name == _LANGGRAPH: - self._read_graphs(folder, path, data) + self._keep_graphs(path, self._graph_targets(folder, data)) else: - self.templates.setdefault(folder, set()).update(_template_names(data)) + self._keep_template(folder, _template_names(data)) + except _CapReached: + self.stopped = "limit" except _FILE_ERRORS: self.unread += 1 except MemoryError: raise except Exception: # A reader fault of a kind not foreseen is still this file's problem, never the - # walk's: the file is unread. The first trace is logged (secret values masked); the rest are - # counted, so a fault that hits every file can't flood the log. + # walk's: the file is unread. The first trace is logged, with only the platform's + # secret values masked; the rest are counted, so a fault that hits every file can't + # flood the log. if not self.faults: logger.warning("agent walk: a file's reader failed", exc_info=True) self.faults += 1 self.unread += 1 - def _read(self, path: str) -> bytes | None: - if self.files_read >= _MAX_FILES: - self.stopped = "limit" - return None - data = _read_file(path, _FILE_BYTES + 1) - self.files_read += 1 - self.bytes_read += len(data) - if self.bytes_read > _MAX_BYTES: - self.stopped = "limit" - return None - if len(data) > _FILE_BYTES: - raise _Unread() - return data - - def _read_graphs(self, folder: str, path: str, data: bytes) -> None: + def _keep_facts(self, path: str, facts: _Facts) -> None: + joined = _joined(path, facts) + if joined and self.joined >= _MAX_JOINED: + raise _CapReached() + self.reader.charge(_held_facts(path, facts)) + self.facts[path] = facts + self._keys[path] = path + self.joined += joined + + def _keep_graphs(self, path: str, targets: list[str]) -> None: + new = [target for target in dict.fromkeys(targets) if target not in self.graphs] + self.reader.charge(sum(_ENTRY + _text(target) + _join_share(target) for target in new)) + for target in new: + self.graphs[target] = path + + def _keep_template(self, folder: str, names: set[str]) -> None: + size = _held_names(names) + if folder not in self.templates: + size += _ENTRY + _text(folder) + _block(_SET) + self.reader.charge(size) + self.templates.setdefault(folder, set()).update(names) + + @staticmethod + def _graph_targets(folder: str, data: bytes) -> list[str]: + """The files a langgraph.json in ``folder`` declares graphs in.""" document = json.loads(data.decode("utf-8")) graphs = document.get("graphs") if isinstance(document, dict) else None if not isinstance(graphs, dict): - return + return [] + targets = [] for declaration in list(graphs.values())[:_NAMES_PER_FILE]: if not isinstance(declaration, str) or ":" not in declaration: continue @@ -899,27 +1110,26 @@ def _read_graphs(self, folder: str, path: str, data: bytes) -> None: file_name = file_name[2:] target = posixpath.normpath(_join(folder, file_name)) if target not in (".", "..") and not target.startswith(("../", "/")): - self.graphs.setdefault(target, path) + targets.append(target) + return targets def _import_roots(self, folder: str) -> list[str]: - """Where an absolute import from ``folder`` is looked for, nearest first.""" - if folder not in self._roots: - roots: list[str] = [] - current = folder - while True: - if not current or self.projects.file_in(current) is not None: - roots += [current, _join(current, "src")] - if not current: - break - current = posixpath.dirname(current) - self._roots[folder] = roots - return self._roots[folder] + """Where an absolute import from ``folder`` is looked for, nearest first: each project + at or above it, and its `src`, then the root's. Worked out from the kept links to the + nearest project, not kept itself.""" + roots: list[str] = [] + current = self.projects.nearest(folder) + while current: + roots += [current, _join(current, "src")] + current = self.projects.nearest(posixpath.dirname(current)) + roots += ["", "src"] + return roots def _module_file(self, base: str, parts: tuple[str, ...]) -> str | None: stem = _join(base, *parts) for path in (f"{stem}.py" if parts else "", _join(stem, "__init__.py")): if path and path in self.facts: - return path + return self._keys[path] return None def _imported(self, path: str) -> tuple[str, ...]: @@ -927,11 +1137,14 @@ def _imported(self, path: str) -> tuple[str, ...]: if path not in self._edges: folder = posixpath.dirname(path) found: dict[str, None] = {} + roots: list[str] | None = None for level, parts in self.facts[path].imports: target = None if level == 0: if parts: - for root in self._import_roots(folder): + if roots is None: + roots = self._import_roots(folder) + for root in roots: target = self._module_file(root, parts) if target is not None: break @@ -1006,34 +1219,38 @@ def agents(self, signs: dict[str, Sign]) -> list[_Agent]: for number, members in enumerate(components) if number not in taken ] - stronger = [ - (self._project_of(path), _is_side_path(path)) - for path, sign in signs.items() - if sign not in _WEAKEST - ] + stronger_any: set[str] = set() + stronger_own: set[str] = set() + for path, sign in signs.items(): + if sign not in _WEAKEST: + project = self._project_of(path) + stronger_any.add(project) + if not _is_side_path(path): + stronger_own.add(project) kept = [] for entry in sorted(entries): self._in_time() side = _is_side_path(entry) if signs[entry] in _WEAKEST: project = self._project_of(entry) - # A side candidate never decides for a file outside side folders. - if any( - other == project and (side or not other_side) - for other, other_side in stronger - ): + if project in (stronger_any if side else stronger_own): continue kept.append((entry, side)) + sharing: dict[str, int] = {} + for other, other_side in kept: + if not other_side: + key = self._project_of(other) + sharing[key] = sharing.get(key, 0) + 1 agents = [] for entry, side in kept: self._in_time() - agents.append(_Agent(entry, signs[entry], side, self._subfolder(entry, side, kept))) + agents.append(_Agent(entry, signs[entry], side, self._subfolder(entry, side, sharing))) return agents def _project_of(self, path: str) -> str: return self.projects.nearest(posixpath.dirname(path)) or "" - def _subfolder(self, entry: str, side: bool, agents: list[tuple[str, bool]]) -> str: + def _subfolder(self, entry: str, side: bool, sharing: dict[str, int]) -> str: folder = posixpath.dirname(entry) if side: return folder @@ -1042,12 +1259,7 @@ def _subfolder(self, entry: str, side: bool, agents: list[tuple[str, bool]]) -> return folder # Only agents of this same project count: one in a deeper project of its own # doesn't move this agent's folder. - sharing = sum( - 1 - for other, other_side in agents - if not other_side and self._project_of(other) == project - ) - return project if sharing == 1 else folder + return project if sharing.get(project, 0) == 1 else folder def candidate( self, agent: _Agent, listed: list[_Agent], *, upload: bool @@ -1061,18 +1273,17 @@ def candidate( for other in listed if other.subfolder != subfolder and _under(other.subfolder, subfolder) ] - own = [agent.entry] if agent.entry in self.facts else [] - for path in self.facts: - if ( - path != agent.entry - and _under(path, subfolder) - and not any(_under(path, folder) for folder in deeper) - and not _is_side_path(_below(path, subfolder)) - ): - own.append(path) - code = sorted({name for path in own for name in self.facts[path].env}) - templates = self.templates.get(subfolder, set()) | self.templates.get("", set()) + names, tools = self._owned(subfolder, deeper) facts = self.facts.get(agent.entry) + if facts is not None and not ( + _under(agent.entry, subfolder) + and not any(_under(agent.entry, folder) for folder in deeper) + and not _is_side_path(_below(agent.entry, subfolder)) + ): + # The entry is always its own row's, even where the folder rule leaves it out. + names = _first_names((names, facts.env)) + tools += facts.tools + templates = self.templates.get(subfolder, set()) | self.templates.get("", set()) project_file = self.projects.file_in(subfolder) return Candidate( entry=agent.entry, @@ -1090,8 +1301,8 @@ def candidate( hazards=self.hazards, ), partial=False, - env_names=(*code, *sorted(templates - set(code))), - tools=sum(self.facts[path].tools for path in own), + env_names=_env_names(names, templates), + tools=tools, prompt_files=_prompt_count(self.prompt_paths, subfolder), evidence=_evidence( agent.entry, @@ -1103,6 +1314,36 @@ def candidate( ), ) + def _owned(self, subfolder: str, deeper: list[str]) -> tuple[list[str], int]: + """The first setting names and the tool count of the kept files a row at ``subfolder`` + owns: under it, under none of the ``deeper`` listed folders, and not in a side folder + below it. Rows of one folder share one pass over its range of the sorted paths.""" + if subfolder in self._owned_by: + return self._owned_by[subfolder] + if self._sorted is None: + self._sorted = sorted(self.facts) + paths = self._sorted + start, end = _range(paths, subfolder) + names: list[str] = [] + tools = 0 + at = start + checked = 0 + for skip_start, skip_end in [*sorted(_range(paths, folder) for folder in deeper), (end, end)]: + while at < min(skip_start, end): + path = paths[at] + at += 1 + checked += 1 + if not checked % 4_096: + self._in_time() + if not _is_side_path(_below(path, subfolder)): + facts = self.facts[path] + if facts.env: + names = _first_names((names, facts.env)) + tools += facts.tools + at = max(at, skip_end) + self._owned_by[subfolder] = (names, tools) + return names, tools + def result(self, *, upload: bool) -> WalkResult: signs: dict[str, Sign] = {} for path, facts in self.facts.items(): @@ -1144,6 +1385,40 @@ def result(self, *, upload: bool) -> WalkResult: ) +def _range(paths: list[str], folder: str) -> tuple[int, int]: + """Where the sorted ``paths`` under ``folder`` are: those that start with ``folder/``, + which sort between it and the same folder ending in the next character, `0`.""" + if not folder: + return 0, len(paths) + return bisect_left(paths, folder + "/"), bisect_left(paths, folder + "0") + + +def _first_names(groups: Iterable[Iterable[str]]) -> list[str]: + """The first `ENV_NAMES_KEPT` distinct names in order, from groups each already in order. + One group stops at its first name past the last kept, so no more than that is held.""" + kept: list[str] = [] + for names in groups: + for name in names: + if len(kept) >= ENV_NAMES_KEPT and name >= kept[-1]: + break + at = bisect_left(kept, name) + if at < len(kept) and kept[at] == name: + continue + kept.insert(at, name) + if len(kept) > ENV_NAMES_KEPT: + kept.pop() + return kept + + +def _env_names(code: list[str], templates: set[str]) -> tuple[str, ...]: + """A row's setting names: the code's first, then a template's that the code doesn't read, + up to `ENV_NAMES_KEPT`, the most the answer file writes. ``code`` is the code's first + names (`_first_names`): when it is short, it is all of them.""" + if len(code) >= ENV_NAMES_KEPT: + return tuple(code[:ENV_NAMES_KEPT]) + return (*code, *sorted(templates - set(code)))[:ENV_NAMES_KEPT] + + def _prompt_count(paths: list[str], subfolder: str) -> int: return sum( 1 @@ -1198,14 +1473,16 @@ def walk_repo( logger.warning("agent walk: joining files into agents ran out of time") walk.facts.clear() return _empty_result(walk, "clock") - except _ResultTooBig: + except (_ResultTooBig, _CapReached): # Too many candidates reach each other to hold: the same end as the walk's other caps. + # The join's lookups all meet folders the walk already charged, so none charges again; + # if one ever did, past a held stop, it ends here too rather than in the crash path. logger.warning("agent walk: joining files into agents passed its edge budget") walk.facts.clear() return _empty_result(walk, "limit") logger.info( "agent walk: files_read=%d unread=%d reader_faults=%d agents=%d left_out=%d stopped=%s", - walk.files_read, + walk.reader.files_read, result.unread_files, walk.faults, len(result.agents), @@ -1226,8 +1503,9 @@ def build_hazard( """Why a build of the folder may stop on its own fallback, or ``""``. ``hazards`` holds one scan's answers by folder, so a folder many agents share is looked - through once. The look stops, with ``""``, at the deadline or after ``_HAZARD_ENTRIES`` - entries: it walks folders the scan skips, which the walk's own clock never saw. + through once. The look takes folders depth first in name order. It stops, with ``""``, at the + deadline, after ``_HAZARD_ENTRIES`` entries, or past ``_HAZARD_PATH_BYTES`` of listed folder + paths: it walks folders the scan skips, which the walk's own clock never saw. """ if hazards is not None and subfolder in hazards: return hazards[subfolder] @@ -1257,18 +1535,25 @@ def _hazard( # folders included, ignoring only its own short list. found = 0 entries = 0 + listed = 0 # os.walk reports no listing errors unless asked, so nothing here is expected to raise. - for index, (_, directories, files) in enumerate(os.walk(folder, followlinks=False)): + for index, (here, directories, files) in enumerate(os.walk(folder, followlinks=False)): # The folder's own listing is one read and always looked at; the clock bounds the # folders below it. if index and clock() > deadline: return "" + # Sorted, so that a look stopped by a cap stops at the same folder on every file system. + directories.sort() + files.sort() directories[:] = [name for name in directories if name not in _BUILD_IGNORED] found += files.count("agent.py") if found > 1: return "agent_py" entries += len(directories) + len(files) - if entries > _HAZARD_ENTRIES: + # What os.walk holds for the folders it has yet to enter: each one's full path, counted in + # characters (a path of wide characters holds up to four bytes for each). + listed += sum(len(here) + 1 + len(name) for name in directories) + if entries > _HAZARD_ENTRIES or listed > _HAZARD_PATH_BYTES: return "" return "" @@ -1295,8 +1580,7 @@ def __init__( self.script_env: set[str] = set() self.templates: set[str] = set() self.prompt_files = 0 - self.files_read = 0 - self.bytes_read = 0 + self.reader = _CappedReader(files=_DESCRIBE_FILES, total_bytes=_DESCRIBE_BYTES) # A cap or the clock: the reading stops here. self.partial = False # One file that couldn't be read or parsed: the counts are short, but the reading @@ -1375,31 +1659,21 @@ def _take(self, directory: str, folder: str, name: str) -> None: self._fault("a file's reader failed") def _read(self, path: str) -> bytes | None: - """The file's bytes, or None for a file that can't or mustn't be read; sets - ``partial`` when a cap is reached.""" - if self.files_read >= _DESCRIBE_FILES: - self.partial = True - return None + """The file's bytes, or None for a file that can't or mustn't be read. Both read sites + (the templates and the source files) come here, outside any parse `try`: a cap sets + ``partial``; a missing file of any kind is nothing; anything else unread is short.""" try: - data = _read_file(path, _FILE_BYTES + 1) + return self.reader.read(path) + except _CapReached: + self.partial = True except FileNotFoundError: # A settings template that isn't there is the usual case, not a short count. - return None - except (OSError, ValueError): - # Unreadable, a link, or a name the system can't open: not read, so the counts - # are short. - self.short = True - return None - self.files_read += 1 - self.bytes_read += len(data) - if self.bytes_read > _DESCRIBE_BYTES: - self.partial = True - return None - if len(data) > _FILE_BYTES: - # Over the size a file is read at: not read, so the counts are short. + pass + except (OSError, ValueError, _Unread): + # Unreadable, a link, a name the system can't open, or over the size a file is + # read at: not read, so the counts are short. self.short = True - return None - return data + return None def describe_entry( @@ -1432,19 +1706,20 @@ def describe_entry( raise except Exception: # Runs after the model was paid for, so a fault in the kit keeps what was read and - # marks the row partial rather than ending in a paid retry. The trace is logged, with the - # platform's secret values masked. + # marks the row partial rather than ending in a paid retry. The trace is logged, with only + # the platform's secret values masked. logger.warning("agent description stopped", exc_info=True) reading.partial = True facts = reading.facts.get(entry) - code = {name for found in reading.facts.values() for name in found.env} - code |= reading.script_env + code = sorted( + {name for found in reading.facts.values() for name in found.env} | reading.script_env + ) project_file = projects.file_in(subfolder) partial = reading.partial or reading.short if not partial: - logger.info("agent description: files_read=%d", reading.files_read) + logger.info("agent description: files_read=%d", reading.reader.files_read) else: - logger.info("agent description: files_read=%d partial=yes", reading.files_read) + logger.info("agent description: files_read=%d partial=yes", reading.reader.files_read) return Candidate( entry=entry, subfolder=subfolder, @@ -1455,7 +1730,7 @@ def describe_entry( source, subfolder, deadline=deadline, clock=clock, hazards=hazards ), partial=partial, - env_names=(*sorted(code), *sorted(reading.templates - code)), + env_names=_env_names(code, reading.templates), tools=sum(found.tools for found in reading.facts.values()), prompt_files=reading.prompt_files, evidence=_evidence( diff --git a/tests/harness/test_scan_folders.py b/tests/harness/test_scan_folders.py index 079dd6666..451dadcdf 100644 --- a/tests/harness/test_scan_folders.py +++ b/tests/harness/test_scan_folders.py @@ -241,6 +241,27 @@ def broken(*args, **kwargs): assert scan_folders._answer(tmp_path, "no/such/folder") == {"name": "unknown", "version": None} +def test_running_out_of_memory_in_the_check_comes_back_as_a_trace(tmp_path, monkeypatch) -> None: + """A MemoryError inside one folder's check is that folder's problem: it comes back with its + trace, as any other fault does, and the child's loop goes on to the next folder. This + differs on purpose from the walk, which re-raises a MemoryError: the walk runs in the scan's + own process, while this runs in the child, where ending it would only lose every later + folder's word. `_serve` leaves its loop only on EOFError or OSError, so an answer here is + the loop going on.""" + import fi.alk.harness.framework_detection as fd + + def out_of_memory(*args, **kwargs): + raise MemoryError() + + monkeypatch.setattr(fd, "folder_findings", out_of_memory) + (tmp_path / "agent.py").write_text(CREW, encoding="utf-8") + + answer = scan_folders._answer(tmp_path, "") + + assert set(answer) == {scan_folders._FAILED} + assert "MemoryError" in answer[scan_folders._FAILED] + + def test_a_failed_answer_marks_the_row_unfinished_and_logs_the_trace(tmp_path, caplog) -> None: """The parent's half: a child answer carrying a trace becomes an unfinished ``unknown``, kept for the folder, with the trace in the log.""" diff --git a/tests/harness/test_scan_walk.py b/tests/harness/test_scan_walk.py index 6231c446f..a781e0381 100644 --- a/tests/harness/test_scan_walk.py +++ b/tests/harness/test_scan_walk.py @@ -1,11 +1,15 @@ from __future__ import annotations import ast +import gc import inspect import io +import itertools import json import os import tokenize +import tracemalloc +from collections.abc import Iterator from pathlib import Path import pytest @@ -342,7 +346,11 @@ def test_a_broken_langgraph_json_is_unread_not_a_crash( def test_the_walk_limit_and_the_clock_stop_it( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - assert (scan_walk._MAX_FILES, scan_walk._MAX_BYTES) == (40_000, 268_435_456) + assert (scan_walk._MAX_JOINED, scan_walk._MAX_BYTES, scan_walk._MAX_HELD) == ( + 40_000, + 268_435_456, + 1_610_612_736, + ) _write(tmp_path, {"a.py": GRAPH, "b.py": GRAPH, "c.py": GRAPH}) calls: list[int] = [] @@ -357,12 +365,12 @@ def clock() -> float: # The walk checked before each of its two files; the join after it checks the clock too. assert len(calls) >= 2 - monkeypatch.setattr(scan_walk, "_MAX_FILES", 2) + monkeypatch.setattr(scan_walk, "_MAX_JOINED", 2) by_count = _walk(tmp_path) assert by_count.stopped == "limit" assert _entries(by_count) == ["a.py", "b.py"] - monkeypatch.setattr(scan_walk, "_MAX_FILES", 40_000) + monkeypatch.setattr(scan_walk, "_MAX_JOINED", 40_000) monkeypatch.setattr(scan_walk, "_MAX_BYTES", len(GRAPH)) by_bytes = _walk(tmp_path) assert by_bytes.stopped == "limit" @@ -1134,6 +1142,53 @@ def counting(top, *args, **kwargs): assert len(entries) <= 3 +def _ordered_walk(real_walk, *, descending: bool, listed: list[str]): + """``os.walk`` with each folder's subfolders handed over in one forced order, recording every + file name it lists.""" + + def walk(*args, **kwargs): + for here, directories, files in real_walk(*args, **kwargs): + directories.sort(reverse=descending) + listed.extend(files) + yield here, directories, files + + return walk + + +@pytest.mark.parametrize("below", ["", "x/"], ids=["top", "nested"]) +@pytest.mark.parametrize("case", ["found", "stopped"]) +def test_a_hazard_look_stopped_by_its_cap_gives_one_answer_in_any_listing_order( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, case: str, below: str +) -> None: + """The look goes in name order whatever order the file system lists folders in, so where its + cap stops it is the same everywhere. Both orders are forced, at the folder the look starts + from and a level below it.""" + if case == "found": + # Name order reaches both agent.py files before the big folder. + _files(tmp_path, {f"{below}a/agent.py": "", f"{below}b/agent.py": ""}) + _files(tmp_path, {f"{below}z/f{index:02}": "" for index in range(50)}) + else: + # Name order meets the big folder first and stops there, before either agent.py. + _files(tmp_path, {f"{below}a/f{index:02}": "" for index in range(50)}) + _files(tmp_path, {f"{below}b/agent.py": "", f"{below}c/agent.py": ""}) + monkeypatch.setattr(scan_walk, "_HAZARD_ENTRIES", 20) + real_walk = os.walk + answers, seen = [], [] + + for descending in (False, True): + listed: list[str] = [] + monkeypatch.setattr(scan_walk.os, "walk", _ordered_walk(real_walk, descending=descending, listed=listed)) + answers.append(scan_walk.build_hazard(tmp_path, "")) + seen.append(listed.count("agent.py")) + + if case == "found": + assert answers == ["agent_py", "agent_py"] + else: + assert answers == ["", ""] + # The "" is the cap's, not a look that missed them: neither agent.py was ever listed. + assert seen == [0, 0] + + def test_the_prompt_files_listed_are_capped( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -1217,9 +1272,9 @@ def test_side_folders_cannot_use_up_the_file_cap_before_the_agent( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """The customer's own code is read first; a big tests folder sorted ahead of the agent - no longer hides it behind the file cap.""" - monkeypatch.setattr(scan_walk, "_MAX_FILES", 10) - _write(tmp_path, {f"tests/test_{index:02}.py": "x = 1\n" for index in range(25)}) + no longer hides it behind the file cap. Each test file imports something, so it counts.""" + monkeypatch.setattr(scan_walk, "_MAX_JOINED", 10) + _write(tmp_path, {f"tests/test_{index:02}.py": "import os\n" for index in range(25)}) _write(tmp_path, {"zz_agent/agent.py": GRAPH}) result = _walk(tmp_path) @@ -1295,8 +1350,8 @@ def counting(self, directory, folder, name): def test_a_template_beside_the_agent_is_read_before_a_big_tests_folder( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - monkeypatch.setattr(scan_walk, "_MAX_FILES", 10) - _write(tmp_path, {f"tests/test_{index:02}.py": "x = 1\n" for index in range(25)}) + monkeypatch.setattr(scan_walk, "_MAX_JOINED", 10) + _write(tmp_path, {f"tests/test_{index:02}.py": "import os\n" for index in range(25)}) _write(tmp_path, {"zz_agent/agent.py": GRAPH, "zz_agent/.env.example": "OPENAI_API_KEY=\n"}) result = _walk(tmp_path) @@ -1391,3 +1446,854 @@ def test_the_step_after_the_walk_holds_a_bounded_number_of_edges( assert bounded.stopped == "limit" assert bounded.agents == () + + +# A UTF-16 byte-order mark followed by an odd number of bytes: the decoder can't read it. The +# same template with its last byte pair whole can. +ODD_UTF16_TEMPLATE = b"\xff\xfeA\x00=\x001\x00\n" +EVEN_UTF16_TEMPLATE = ODD_UTF16_TEMPLATE + b"\x00" + + +def test_a_template_the_decoder_rejects_marks_the_row_short( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + with pytest.raises(UnicodeDecodeError): + scan_walk._template_names(ODD_UTF16_TEMPLATE) + # The entry, and a file after the template that is still read: its setting is kept. + _write(tmp_path, {"app/bot.py": CLIENT, "app/tool.py": "import os\nos.getenv('TOOL_KEY')\n"}) + (tmp_path / "app" / ".env.example").write_bytes(EVEN_UTF16_TEMPLATE) + readable = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) + (tmp_path / "app" / ".env.example").write_bytes(ODD_UTF16_TEMPLATE) + + with caplog.at_level("WARNING", logger=scan_walk.logger.name): + rejected = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) + + assert readable is not None and readable.partial is False + assert set(readable.env_names) == {"A", "TOOL_KEY"} + assert rejected is not None and rejected.partial is True + assert set(rejected.env_names) == {"TOOL_KEY"} + # Short because the decoder rejected it, not because the kit's reader failed. + assert "agent description:" not in caplog.text + + +def test_the_walks_summary_line_counts_reader_faults_apart_from_unread_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + _write(tmp_path, {f"m{index}.py": "x = 1\n" for index in range(3)}) + # An ordinary unreadable file: the tokenizer rejects it, which is no fault of the kit's. + _write(tmp_path, {"unterminated.py": "x = '''never closed\n"}) + real = scan_walk._python_facts + + def broken(data: bytes, shadowed: frozenset[str]) -> object: + if data == b"x = 1\n": + raise LookupError("a fault in the kit's reader") + return real(data, shadowed) + + monkeypatch.setattr(scan_walk, "_python_facts", broken) + with caplog.at_level("INFO", logger=scan_walk.logger.name): + _walk(tmp_path) + + assert "unread=4 reader_faults=3 " in caplog.text + + +def test_a_fault_in_the_description_keeps_what_was_read_and_logs_the_trace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """The description runs after the model was paid for, so a fault in the kit that escapes + it keeps what was read, as a partial row, instead of ending the scan in a paid retry. The + fault raised is outside the per-file catch, so only `describe_entry`'s own catch holds it.""" + _write( + tmp_path, + { + "app/bot.py": CLIENT, + "app/a_first.py": "import os\nos.getenv('FIRST_KEY')\n", + "app/z_last.py": "x = 1\n", + }, + ) + take = scan_walk._Description._take + calls: list[str] = [] + + def failing_after_the_first(self: object, directory: str, folder: str, name: str) -> None: + calls.append(name) + if len(calls) > 1: + raise RuntimeError("a fault in the kit") + take(self, directory, folder, name) # type: ignore[arg-type] + + monkeypatch.setattr(scan_walk._Description, "_take", failing_after_the_first) + with caplog.at_level("WARNING", logger=scan_walk.logger.name): + candidate = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) + + assert calls[0] == "a_first.py" + assert candidate is not None + assert (candidate.entry, candidate.subfolder, candidate.partial) == ("app/bot.py", "app", True) + assert "FIRST_KEY" in candidate.env_names + assert "agent description stopped" in caplog.text + assert "RuntimeError: a fault in the kit" in caplog.text + + +# -- what the walk and its join keep (TH-9449), and its one capped reader (TH-9450) ---------- + + +def _unique_parts() -> Iterator[str]: + first = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_" + rest = first + "0123456789" + return ("".join(name) for name in itertools.product(first, rest, rest)) + + +def _crafted(parts: Iterator[str]) -> str: + """200 imports of 64 unique three-letter parts: the most kept for the bytes read.""" + return "".join("import " + ".".join(next(parts) for _ in range(64)) + "\n" for _ in range(200)) + + +def _files(root: Path, files: dict[str, str]) -> None: + for relative, content in files.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def _shape(root: Path, shape: str) -> None: + """Sixteen repository shapes, small, each growing one thing the walk or its join keeps.""" + if shape == "empty": + _files(root, {f"p{index % 3}/m{index:03}.py": "" for index in range(300)}) + elif shape == "plain_imports": + # Nine-character names: the allocator gives each 64 bytes, more than its size asked for. + _files(root, {f"m{i:02}.py": "".join(f"import mod{i:02}_{n:03}\n" for n in range(200)) for i in range(60)}) + elif shape == "dotted_imports": + _files( + root, + {f"m{i:02}.py": "".join(f"import pkg{i}_{n}.sub{n}.mod{n}\n" for n in range(200)) for i in range(60)}, + ) + elif shape == "from_import": + _files( + root, + { + f"m{i:02}.py": "from pkg.mod import (" + ", ".join(f"name_{i}_{n}" for n in range(300)) + ")\n" + for i in range(60) + }, + ) + elif shape == "long_names": + _files( + root, + { + f"m{i:02}.py": "".join( + "import " + ".".join([f"n{i}_{n}" + "q" * 250] * 16) + "\n" for n in range(200) + ) + for i in range(20) + }, + ) + elif shape == "settings": + _files( + root, + { + f"m{i:02}.py": "import os\n" + "".join(f"os.getenv('E{i}_{n}_{'X' * 120}')\n" for n in range(200)) + for i in range(60) + }, + ) + elif shape == "wide_names": + _files( + root, + { + f"m{i:02}.py": "".join(f"import 変数変数変数変数{i}_{n}.модульмодуль{n}\n" for n in range(200)) + for i in range(60) + }, + ) + elif shape == "crafted": + parts = _unique_parts() + _files(root, {f"m{i:02}.py": _crafted(parts) for i in range(12)}) + elif shape == "templates": + _files( + root, + {f"f{i:03}/.env.example": "".join(f"S{i}_{n}_NAME=\n" for n in range(200)) for i in range(200)}, + ) + elif shape == "graphs": + _files( + root, + { + f"g{i:02}/langgraph.json": json.dumps( + {"graphs": {f"x{n}": f"./deep/m{n}_{'y' * 40}.py:graph" for n in range(200)}} + ) + for i in range(20) + }, + ) + elif shape == "comb_and_trunk": + _files(root, {"/".join(f"c{c}d{d}" for d in range(100)) + "/m.py": "" for c in range(40)}) + trunk = "/".join(f"t{d}" for d in range(100)) + for depth in range(1, 101): + _files(root, {"/".join(trunk.split("/")[:depth]) + "/pyproject.toml": ""}) + _files(root, {f"{trunk}/leaf{n}/bot.py": "import os\nOpenAI()\n" for n in range(20)}) + elif shape == "importers_and_candidates": + deep = "/".join(f"level{d:02}_{'z' * 20}" for d in range(30)) + _files(root, {f"{deep}/mod{n:03}.py": "" for n in range(200)}) + _files( + root, + {f"{deep}/imp{n:02}.py": "".join(f"from . import mod{k:03}\n" for k in range(200)) for n in range(50)}, + ) + # Candidates that reach the importers, and many that reach nothing: the join's shares + # and import lists outweigh its fixed part. + _files(root, {f"{deep}/c{n:02}.py": f"from . import imp{n:02}\nOpenAI()\n" for n in range(50)}) + _files(root, {f"tiny/c{n:04}.py": "OpenAI()\n" for n in range(5_000)}) + elif shape == "project_per_file": + for index in range(300): + _files(root, {f"proj{index:03}/pyproject.toml": "", f"proj{index:03}/m.py": "x = 1\n"}) + elif shape == "rows_and_broom": + _files(root, {f"a{n:02}.py": "OpenAI()\n" for n in range(50)}) + _files( + root, + { + f"lib/m{i:03}.py": "import os\n" + "".join(f"os.getenv('L{i}_{n}')\n" for n in range(100)) + for i in range(100) + }, + ) + chain = root / "broom" + for depth in range(50): + chain = chain / f"d{depth}" + for sibling in range(20): + (chain.parent / f"s{depth}_{sibling}").mkdir(parents=True, exist_ok=True) + chain.mkdir(parents=True, exist_ok=True) + elif shape == "one_name_files": + _files(root, {"a.py": "OpenAI()\n"}) + _files(root, {f"n/f{n:04}.py": f"os.getenv('N{n:05}')\n" for n in range(2_000)}) + elif shape == "levels_over_256": + # Each import's level is an int of its own, and each client's line number too. + dots = "." * 300 + _files( + root, + { + f"m{i:02}.py": "".join(f"from {dots}m{n} import x\n" for n in range(200)) + "OpenAI()\n" * 100 + for i in range(60) + }, + ) + else: + raise AssertionError(shape) + + +SHAPES = [ + "empty", + "plain_imports", + "dotted_imports", + "from_import", + "long_names", + "settings", + "wide_names", + "crafted", + "templates", + "graphs", + "comb_and_trunk", + "importers_and_candidates", + "project_per_file", + "rows_and_broom", + "one_name_files", + "levels_over_256", +] +ORDINARY = SHAPES[1:7] + + +def _allocated(size: int) -> int: + """The block a request of ``size`` gets: pymalloc's 16-byte classes up to 512, then glibc's + malloc (an 8-byte header, 16-byte alignment, 32 at least), as on the Linux sandbox. Kept + apart from the code's own `_block`, so a fault in that can't hide here.""" + if size <= 512: + return max(16, (size + 15) // 16 * 16) + return max(32, (size + 8 + 15) // 16 * 16) + + +def _rounded_live() -> int: + return sum(_allocated(trace.size) for trace in tracemalloc.take_snapshot().traces) + + +def _largest_parse_peak(root: Path) -> int: + files = sorted(root.rglob("*.py"), key=lambda path: path.stat().st_size) + if not files: + return 0 + tracemalloc.start() + try: + base = tracemalloc.get_traced_memory()[0] + tracemalloc.reset_peak() + scan_walk._python_facts(files[-1].read_bytes(), frozenset()) + return tracemalloc.get_traced_memory()[1] - base + finally: + tracemalloc.stop() + + +def _hazard_peak(root: Path) -> int: + """A build hazard's own look at the root, the biggest any row's can be: bounded on its own + (`_HAZARD_PATH_BYTES`), not charged.""" + tracemalloc.start() + try: + base = tracemalloc.get_traced_memory()[0] + tracemalloc.reset_peak() + scan_walk.build_hazard(root, "") + return tracemalloc.get_traced_memory()[1] - base + finally: + tracemalloc.stop() + + +def _measure(root: Path) -> dict[str, int]: + """Walk and join ``root`` under tracemalloc: what was charged, what is really kept.""" + gc.collect() + tracemalloc.start() + try: + walk = scan_walk._Walk(root, deadline=FOREVER, clock=lambda: 0.0) + walk.run() + walk_peak = tracemalloc.get_traced_memory()[1] + gc.collect() + after_walk = tracemalloc.get_traced_memory()[0] + tracemalloc.reset_peak() + result = walk.result(upload=False) + join_peak = tracemalloc.get_traced_memory()[1] + gc.collect() + live = _rounded_live() + finally: + tracemalloc.stop() + join_charged = scan_walk._JOIN_BASE + join_charged += sum(scan_walk._held_for_join(path, facts) for path, facts in walk.facts.items()) + join_charged += sum(scan_walk._join_share(target) for target in walk.graphs) + assert result.stopped == "" + return { + "charged": walk.reader.held, + "live": live, + "walk_peak": walk_peak, + "join_rise": join_peak - after_walk, + "join_charged": join_charged, + } + + +@pytest.mark.parametrize("shape", SHAPES) +def test_what_is_charged_is_never_below_what_the_walk_and_its_join_keep( + tmp_path: Path, shape: str +) -> None: + """Checked against tracemalloc on CPython 3.12, the kit's Python, with each live block + rounded as the allocator rounds it (`_block`). The sizes in the estimate are 3.12's.""" + _shape(tmp_path, shape) + parse_peak = _largest_parse_peak(tmp_path) + hazard_peak = _hazard_peak(tmp_path) + + seen = _measure(tmp_path) + + assert seen["live"] <= seen["charged"], seen + assert seen["join_rise"] <= seen["join_charged"] + hazard_peak, (seen, hazard_peak) + assert seen["walk_peak"] <= seen["charged"] + parse_peak + 1_048_576, seen + if shape in ORDINARY: + # High, but not so high that an ordinary repository is cut early. + assert seen["charged"] <= 2.5 * seen["live"], seen + + +def test_the_walk_stops_at_what_it_keeps_not_at_how_many_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + parts = _unique_parts() + crafted = _crafted(parts) + _files(tmp_path / "crafted", {f"m{index:02}.py": _crafted(parts) for index in range(12)}) + _files(tmp_path / "small", {f"m{index:02}.py": "" for index in range(12)}) + one = scan_walk._held_facts("m00.py", scan_walk._python_facts(crafted.encode(), frozenset())) + monkeypatch.setattr(scan_walk, "_MAX_HELD", scan_walk._JOIN_BASE + 6 * one) + + stopped = _walk(tmp_path / "crafted") + whole = _walk(tmp_path / "small") + + assert stopped.stopped == "limit" + # The file whose charge crossed the budget is kept nowhere and isn't counted unread. + assert stopped.unread_files == 0 + assert whole.stopped == "" + + +def test_many_small_files_are_read_whole_and_only_joined_files_count( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A 40,000-file cap used to stop here, with the agent the last file read.""" + for folder in range(201): + package = tmp_path / "small" / f"p{folder:03}" + package.mkdir(parents=True) + for index in range(200): + (package / f"m{index:03}.py").touch() + _files(tmp_path / "small", {"zz_agent/agent.py": GRAPH}) + _files(tmp_path / "importing", {f"m{index:02}.py": "import os\n" for index in range(12)}) + + with caplog.at_level("INFO", logger=scan_walk.logger.name): + small = _walk(tmp_path / "small") + monkeypatch.setattr(scan_walk, "_MAX_JOINED", 10) + importing = _walk(tmp_path / "importing") + + assert small.stopped == "" + assert _entries(small) == ["zz_agent/agent.py"] + assert "files_read=40201 " in caplog.text + assert importing.stopped == "limit" + + +# For each thing kept: two repositories that differ only in it, and which measure shows it: +# what is live after the join, the walk's peak (the folders `os.walk` holds until it enters +# them), or the join's own rise (what the join builds for the files it reaches). +KEPT = { + "templates": "live", + "graphs": "live", + "prompt_paths": "live", + "project_sets": "live", + "pending_folders": "walk_peak", + "edges": "join", + "join_share": "join", + "import_levels": "live", + "line_numbers": "live", +} + + +def _kept_pair(with_it: Path, without: Path, structure: str) -> None: + if structure == "templates": + _files(with_it, {f"f{i:02}/.env.example": "".join(f"N{i}_{n}=\n" for n in range(100)) for i in range(40)}) + _files(without, {f"f{i:02}/readme.txt": "x\n" for i in range(40)}) + elif structure == "graphs": + graphs = {"graphs": {f"g{n}": f"./m{n}_{'y' * 30}.py:graph" for n in range(200)}} + _files(with_it, {f"a{i}/langgraph.json": json.dumps(graphs) for i in range(10)}) + _files(without, {f"a{i}/langgraph.json": "{}" for i in range(10)}) + elif structure == "prompt_paths": + _files(with_it, {f"prompts/prompt_{n:04}_{'p' * 60}.txt": "" for n in range(500)}) + _files(without, {f"others/other_{n:04}_{'p' * 60}.txt": "" for n in range(500)}) + elif structure == "project_sets": + for index in range(200): + _files(with_it, {f"p{index:03}/pyproject.toml": "", f"p{index:03}/m.py": "", f"p{index:03}/n.py": ""}) + _files(without, {f"p{index:03}/m.py": "", f"p{index:03}/n.py": ""}) + elif structure == "pending_folders": + for root, siblings in ((with_it, 20), (without, 0)): + chain = root + for depth in range(40): + chain = chain / f"d{depth}" + for sibling in range(siblings): + (chain.parent / f"s{depth}_{sibling}_{'w' * 30}").mkdir(parents=True) + chain.mkdir(parents=True) + elif structure == "edges": + # One candidate reaches 200 files that each import the other 199: the join keeps an + # import list for each. Without, the same files import nothing. + web = {f"m{n:03}.py": "".join(f"import m{k:03}\n" for k in range(200) if k != n) for n in range(200)} + _files(with_it, {"bot.py": "import m000\nOpenAI()\n", **web}) + _files(without, {"bot.py": "OpenAI()\n", **{name: "" for name in web}}) + elif structure == "import_levels": + # The same imports; only their levels pass 256, so each is an int of its own rather than + # a shared small one. + _files(with_it, {f"m{i:02}.py": "".join(f"from {'.' * 300}m{n} import x\n" for n in range(200)) for i in range(30)}) + _files(without, {f"m{i:02}.py": "".join(f"from .m{n} import x\n" for n in range(200)) for i in range(30)}) + elif structure == "line_numbers": + # One client a file, on a line past 256 or on line 1: an int of its own, or a shared one. + _files(with_it, {f"c{n:04}.py": "\n" * 300 + "OpenAI()\n" for n in range(2_000)}) + _files(without, {f"c{n:04}.py": "OpenAI()\n" + "\n" * 300 for n in range(2_000)}) + else: + _files(with_it, {f"c{n:04}.py": "OpenAI()\n" for n in range(5_000)}) + _files(without, {f"c{n:04}.py": "x = 1\n" for n in range(5_000)}) + + +@pytest.mark.parametrize("structure", list(KEPT)) +def test_each_thing_kept_is_charged_at_least_what_it_holds(tmp_path: Path, structure: str) -> None: + _kept_pair(tmp_path / "with", tmp_path / "without", structure) + + a, b = _measure(tmp_path / "with"), _measure(tmp_path / "without") + + if KEPT[structure] == "join": + # Charged at the walk for every file, whether the join reaches it or not, so the + # repository with the thing is checked on its own. + assert a["join_rise"] - b["join_rise"] > 0, (a, b) + assert a["join_rise"] <= a["join_charged"], a + else: + measure = KEPT[structure] + assert a["charged"] - b["charged"] >= a[measure] - b[measure] > 0, (a, b) + + +def _all_names(walk: scan_walk._Walk, subfolder: str) -> list[str]: + """What today's code built: every distinct name under the row's folder, sorted.""" + return sorted({name for path, facts in walk.facts.items() if scan_walk._under(path, subfolder) for name in facts.env}) + + +@pytest.mark.parametrize("names", [30, 300]) +def test_a_rows_setting_names_are_its_first_fifty(tmp_path: Path, names: int) -> None: + _files(tmp_path, {"bot.py": "from openai import OpenAI\nOpenAI()\n"}) + _files( + tmp_path, + {f"lib/m{i:02}.py": "import os\n" + "".join(f"os.getenv('K{(i * 37 + n) % names:04}')\n" for n in range(40)) for i in range(20)}, + ) + _files(tmp_path, {".env.example": "".join(f"T{n:03}=\n" for n in range(40))}) + walk = scan_walk._Walk(tmp_path, deadline=FOREVER, clock=lambda: 0.0) + walk.run() + full = _all_names(walk, "") + expected = (*full, *sorted(walk.templates[""] - set(full)))[: scan_walk.ENV_NAMES_KEPT] + + row = _agent(walk.result(upload=False), "bot.py") + described = describe_entry(tmp_path, "bot.py", "", deadline=FOREVER) + + assert row.env_names == expected + assert described is not None and described.env_names == expected + assert scan_walk.ENV_NAMES_KEPT == 50 + from fi.alk.harness import scan_document + + assert scan_document.ENV_NAMES_KEPT is scan_walk.ENV_NAMES_KEPT + + +def test_a_hazard_look_gives_up_past_its_listed_paths(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _files(tmp_path, {"a/agent.py": "", "b/agent.py": ""}) + # Beside a and b, so the root's own listing passes the bound before either agent.py is + # seen, whatever order the file system lists folders in. + for index in range(40): + (tmp_path / f"folder_{index:02}_{'v' * 40}").mkdir() + assert scan_walk.build_hazard(tmp_path, "") == "agent_py" + + monkeypatch.setattr(scan_walk, "_HAZARD_PATH_BYTES", 500) + + assert scan_walk.build_hazard(tmp_path, "") == "" + + +@pytest.mark.parametrize( + ("weak", "stronger", "listed"), + [ + ("tests/test_client.py", "tests/test_graph.py", False), + ("app/client.py", "tests/test_graph.py", True), + ("app/client.py", "app/graph.py", False), + ], + ids=["side_beside_side", "own_beside_side_only", "own_beside_own"], +) +def test_a_side_candidate_never_decides_for_one_outside_side_folders( + tmp_path: Path, weak: str, stronger: str, listed: bool +) -> None: + _files(tmp_path, {"pyproject.toml": "", weak: "from openai import OpenAI\nOpenAI()\n", stronger: GRAPH}) + + assert (weak in _entries(_walk(tmp_path))) is listed + + +def test_the_join_is_linear_in_its_candidates(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _files(tmp_path, {"pyproject.toml": "", **{f"c{n:04}.py": "OpenAI()\n" for n in range(2_000)}}) + _files(tmp_path, {f"lib/m{n:04}.py": "" for n in range(2_000)}) + walk = scan_walk._Walk(tmp_path, deadline=FOREVER, clock=lambda: 0.0) + walk.run() + project_of = scan_walk._Walk._project_of + side_checks = scan_walk._is_side_path + counted = {"project_of": 0, "side": 0} + + def counting_project_of(self: object, path: str) -> str: + counted["project_of"] += 1 + return project_of(self, path) # type: ignore[arg-type] + + def counting_side(path: str) -> bool: + counted["side"] += 1 + return side_checks(path) + + monkeypatch.setattr(scan_walk._Walk, "_project_of", counting_project_of) + monkeypatch.setattr(scan_walk, "_is_side_path", counting_side) + result = walk.result(upload=False) + + assert len(result.agents) == 50 + # Today's loops asked for each candidate's project once per other candidate, and walked + # every kept file once per row: millions here. + assert counted["project_of"] <= 3 * 2_000 + assert counted["side"] <= 10 * 4_000 + + +def test_a_folder_and_its_ancestors_are_looked_at_once(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + # 150 levels: macOS paths stop at 1,024 characters. + deep = "/".join(f"d{depth}" for depth in range(150)) + _files(tmp_path, {f"{deep}/m{n:03}.py": f"import m{(n + 1) % 300:03}\n" for n in range(300)}) + file_in = scan_walk._Projects.file_in + looks: list[str] = [] + + def counting(self: object, folder: str) -> str | None: + looks.append(folder) + return file_in(self, folder) # type: ignore[arg-type] + + monkeypatch.setattr(scan_walk._Projects, "file_in", counting) + _walk(tmp_path) + + # 151 folders, the root included; a look a file at a time would be 300 times as many. + assert len(looks) <= 2 * 151 + assert len(set(looks)) == 151 + + +def test_the_capped_reader_counts_in_the_order_the_stop_points_need(tmp_path: Path) -> None: + for name, size in (("a", 6), ("b", 6), ("big", scan_walk._FILE_BYTES + 1)): + (tmp_path / name).write_bytes(b"x" * size) + missing = str(tmp_path / "missing") + + by_files = scan_walk._CappedReader(files=2, total_bytes=10**9) + assert by_files.read(str(tmp_path / "a")) == b"x" * 6 + with pytest.raises(FileNotFoundError): + by_files.read(missing) + assert by_files.files_read == 1 # a file that can't be read isn't counted + by_files.read(str(tmp_path / "b")) + with pytest.raises(scan_walk._CapReached): + by_files.read(missing) # the count is checked before reading + + by_bytes = scan_walk._CappedReader(files=None, total_bytes=10) + by_bytes.read(str(tmp_path / "a")) + with pytest.raises(scan_walk._CapReached): + by_bytes.read(str(tmp_path / "b")) # read and counted, then past the bytes + assert (by_bytes.files_read, by_bytes.bytes_read) == (2, 12) + + by_size = scan_walk._CappedReader(files=None, total_bytes=10**9) + with pytest.raises(scan_walk._Unread): + by_size.read(str(tmp_path / "big")) + assert by_size.files_read == 1 + + by_held = scan_walk._CappedReader(files=None, total_bytes=10**9, held=10) + by_held.charge(10) + with pytest.raises(scan_walk._CapReached): + by_held.charge(1) + scan_walk._CappedReader(files=None, total_bytes=10**9).charge(10**12) # no budget: never + assert not issubclass(scan_walk._CapReached, scan_walk._FILE_ERRORS) + + +def test_the_walk_and_the_description_read_through_the_one_capped_reader(tmp_path: Path) -> None: + walk = scan_walk._Walk(tmp_path, deadline=FOREVER, clock=lambda: 0.0) + description = scan_walk._Description(os.fspath(tmp_path), "bot.py", "", deadline=FOREVER, clock=lambda: 0.0) + + assert isinstance(walk.reader, scan_walk._CappedReader) + assert isinstance(description.reader, scan_walk._CappedReader) + for reading in (scan_walk._Walk, scan_walk._Description): + assert "_read_file(" not in inspect.getsource(reading), reading.__name__ + + +@pytest.mark.parametrize("where", ["template", "source"]) +def test_the_descriptions_cap_stops_it_partial_at_either_read_site( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, where: str +) -> None: + _files(tmp_path, {"app/bot.py": CLIENT, "app/tool.py": "x = 1\n"}) + if where == "template": + # The cap is checked before each read: here before the first template is looked at. + _files(tmp_path, {"app/.env.example": "A=1\n"}) + monkeypatch.setattr(scan_walk, "_DESCRIBE_FILES", 0) + else: + # No template is there, so the first read counted is bot.py; tool.py is past the cap. + monkeypatch.setattr(scan_walk, "_DESCRIBE_FILES", 1) + # Past the cap the description reads nothing more: these come after it. + _files(tmp_path, {f"app/prompts/prompt_{n}.md": "Be brief.\n" for n in range(7)}) + + with caplog.at_level("WARNING", logger=scan_walk.logger.name): + candidate = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) + + assert candidate is not None and candidate.partial is True + assert not candidate.prompt_files + assert "agent description stopped" not in caplog.text + + +def test_a_rows_pass_over_many_files_watches_the_clock( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """One row's pass over the kept files checks the clock as it goes, so a huge tree of files + that import nothing ends as a clock stop, never past the helper's hard stop.""" + _files(tmp_path, {"bot.py": "from openai import OpenAI\nOpenAI()\n"}) + for folder in range(50): + package = tmp_path / f"p{folder:02}" + package.mkdir() + for index in range(200): + (package / f"m{index:03}.py").touch() + now = [0.0] + side_checks = scan_walk._is_side_path + seen = [0] + + def counting(path: str) -> bool: + seen[0] += 1 + if seen[0] == 5_000: + now[0] = 10_000.0 # past the result's deadline, mid-pass + return side_checks(path) + + walk = scan_walk._Walk(tmp_path, deadline=100.0, clock=lambda: now[0]) + walk.run() + assert walk.stopped == "" + monkeypatch.setattr(scan_walk, "_is_side_path", counting) + + with pytest.raises(scan_walk._ResultOutOfTime): + walk.result(upload=False) + + +def test_a_project_lookup_is_charged_at_least_what_it_keeps(tmp_path: Path) -> None: + """Each folder the lookup keeps, with its link to its nearest project, measured on the + lookup alone: the walk also charges these folders as ones to enter, which would hide it.""" + folders = ["/".join(f"c{c}d{d}" for d in range(60)) for c in range(30)] + for folder in folders: + (tmp_path / folder).mkdir(parents=True) + (tmp_path / "pyproject.toml").touch() + charged = [0] + + def charge(size: int) -> None: + charged[0] += size + + projects = scan_walk._Projects(os.fspath(tmp_path), scan_walk._PROJECT_FILES, charge) + gc.collect() + tracemalloc.start() + try: + for folder in folders: + projects.nearest(folder) + gc.collect() + kept = _rounded_live() + finally: + tracemalloc.stop() + + assert projects.nearest(folders[0]) == "" + assert kept <= charged[0] + + +def test_the_first_names_hold_no_more_than_fifty_and_stop_early() -> None: + names = [f"N{n:04}" for n in range(300)] + taken: list[str] = [] + + def group(start: int) -> Iterator[str]: + for name in names[start:]: + taken.append(name) + yield name + + first = scan_walk._first_names([group(100), group(0), group(250)]) + + assert first == names[:50] + # Each group stops at its first name past the last of 50 kept: the first fills to 50 and + # stops at its 51st, the second replaces them all and stops at its 51st, and the third's + # first name is already past. + assert len(taken) == 51 + 51 + 1 + + +def test_a_rows_names_leave_out_a_deeper_listed_folder_and_a_sibling_with_the_same_start( + tmp_path: Path, +) -> None: + _files( + tmp_path, + { + "bot.py": GRAPH + "import os\nos.getenv('ROOT_KEY')\n", + "app/pyproject.toml": "", + "app/agent.py": GRAPH + "import os\nos.getenv('APP_KEY')\n", + # Not under app/, though its name starts the same way. + "app2/helper.py": "import os\nos.getenv('APP2_KEY')\n", + "plugins/x/pyproject.toml": "", + "plugins/x/agent.py": GRAPH + "import os\nos.getenv('PLUGIN_KEY')\n", + }, + ) + + result = _walk(tmp_path) + by_folder = {agent.subfolder: set(agent.env_names) for agent in result.agents} + + assert by_folder == { + "": {"ROOT_KEY", "APP2_KEY"}, + "app": {"APP_KEY"}, + "plugins/x": {"PLUGIN_KEY"}, + } + + +def test_the_estimate_rounds_each_object_up_to_its_block() -> None: + """The estimate's sizes are 3.12's, each rounded as the allocator rounds it. The whole-walk + comparison has margins beside the rounding, so the rounding is held here on its own.""" + import sys + + for size in range(1, 4_097): + assert scan_walk._block(size) >= _allocated(size), size + for name in ("", "a", "abcdefgh", "x" * 255, "変数", "модуль", "\U0001d400" * 3): + assert scan_walk._text(name) >= _allocated(sys.getsizeof(name)), name + for items in range(0, 210): + assert scan_walk._tuple(items) >= _allocated(sys.getsizeof(tuple(range(items)))), items + + +@pytest.mark.parametrize("structure", ["pending_folders", "prompt_paths"]) +def test_a_charge_made_outside_one_files_read_stops_the_walk_at_limit( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, structure: str +) -> None: + """A folder's listing and a prompt path are charged outside any one file's read, so only the + walk's own catch turns their cap into `limit`: the walk keeps the row it found and never + raises out of `walk_repo`.""" + _files(tmp_path, {"bot.py": CLIENT}) + (tmp_path / "z").mkdir() + before = scan_walk._Walk(tmp_path, deadline=FOREVER, clock=lambda: 0.0) + before.run() + for index in range(300): + if structure == "pending_folders": + (tmp_path / "z" / f"s{index:03}_{'w' * 30}").mkdir() + else: + (tmp_path / "z" / f"prompt_{index:03}.txt").touch() + monkeypatch.setattr(scan_walk, "_MAX_HELD", before.reader.held + 1_024) + + result = _walk(tmp_path) + + assert result.stopped == "limit" + assert _entries(result) == ["bot.py"] + + +def test_a_descriptions_byte_cap_marks_it_partial(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _files(tmp_path, {"app/bot.py": CLIENT, "app/tool.py": "x = 1\n" * 50}) + whole = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) + monkeypatch.setattr(scan_walk, "_DESCRIBE_BYTES", len(CLIENT) + 10) + + capped = describe_entry(tmp_path, "app/bot.py", "app", deadline=FOREVER) + + assert whole is not None and whole.partial is False + assert capped is not None and capped.partial is True + + +def test_a_project_lookup_charges_each_folder_and_each_link_it_keeps(tmp_path: Path) -> None: + """Every folder looked at is charged once for its entry, and every link to a nearest project + once more; the first look at a deep folder keeps a link for each ancestor, so a file below + one of them is answered from the links.""" + deep = "/".join(f"d{depth}" for depth in range(40)) + (tmp_path / deep).mkdir(parents=True) + (tmp_path / "d0" / "d1" / "pyproject.toml").touch() + charges: list[int] = [] + projects = scan_walk._Projects(os.fspath(tmp_path), scan_walk._PROJECT_FILES, charges.append) + + assert projects.nearest(deep) == "d0/d1" + + ancestors = [deep.rsplit("/", cut)[0] for cut in range(1, 40)] + [""] + assert set(projects._nearest) == {deep, *ancestors} + assert sum(charges) == sum(scan_walk._ENTRY + scan_walk._text(folder) for folder in projects._file) + ( + scan_walk._ENTRY * len(projects._nearest) + ) + looked = len(charges) + assert projects.nearest(ancestors[10]) == "d0/d1" + assert len(charges) == looked + + +def test_a_hazard_look_counts_each_listed_folder_by_its_full_path( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _files(tmp_path, {"a/agent.py": "", "b/agent.py": ""}) + names = [f"folder_{index:02}" for index in range(40)] + for name in names: + (tmp_path / name).mkdir() + # Above the names alone, below the names with the folder they sit in: the root's listing alone + # passes it, before either agent.py is found. + monkeypatch.setattr(scan_walk, "_HAZARD_PATH_BYTES", sum(len(name) + 1 for name in [*names, "a", "b"]) + 100) + + assert scan_walk.build_hazard(tmp_path, "") == "" + + +def test_the_file_whose_charge_crosses_the_budget_is_kept_nowhere( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _files(tmp_path, {"a.py": GRAPH, "b.py": GRAPH, "c.py": GRAPH}) + whole = scan_walk._Walk(tmp_path, deadline=FOREVER, clock=lambda: 0.0) + whole.run() + # One byte short of the whole walk's charge: the last file's charge crosses it. + monkeypatch.setattr(scan_walk, "_MAX_HELD", whole.reader.held - 1) + + walk = scan_walk._Walk(tmp_path, deadline=FOREVER, clock=lambda: 0.0) + walk.run() + result = _walk(tmp_path) + + assert walk.stopped == "limit" + assert len(walk.facts) == len(walk._keys) == 2 + assert result.stopped == "limit" and len(_entries(result)) == 2 + + +def test_an_import_inside_an_inner_project_resolves_at_an_outer_projects_root(tmp_path: Path) -> None: + """The places an import is looked for are every project at or above the file, nearest + first, then the repository's root: here the helper is found at the outer project's root, so + it joins the graph's agent rather than standing as one of its own.""" + _files( + tmp_path, + { + "svc/pyproject.toml": "", + "svc/a/pyproject.toml": "", + "svc/a/graph.py": GRAPH + "from helpers.llm import ask\n", + "svc/helpers/llm.py": CLIENT + "def ask():\n pass\n", + }, + ) + + assert _entries(_walk(tmp_path)) == ["svc/a/graph.py"] + + +def test_a_cap_reached_while_joining_ends_as_limit(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _files(tmp_path, {"bot.py": CLIENT}) + + def charging(self: object, **kwargs: object) -> object: + raise scan_walk._CapReached() + + monkeypatch.setattr(scan_walk._Walk, "result", charging) + + assert _walk(tmp_path).stopped == "limit"