diff --git a/.github/workflows/sdk-smoke.yml b/.github/workflows/sdk-smoke.yml index 8589e0e0d..8a330beb9 100644 --- a/.github/workflows/sdk-smoke.yml +++ b/.github/workflows/sdk-smoke.yml @@ -90,6 +90,8 @@ jobs: tests/harness/test_process_preflight.py tests/providers tests/harness/test_scenario_source_provision_agent.py + tests/harness/test_outbound.py + tests/harness/test_contract_writer.py tests/harness/test_framework_detection.py tests/harness/test_hosted_entrypoint.py tests/harness/test_hosted_entrypoint_provision_retry.py diff --git a/src/fi/alk/harness/hosted_chat_entrypoint.py b/src/fi/alk/harness/hosted_chat_entrypoint.py index c1a300d22..41f8cc23d 100644 --- a/src/fi/alk/harness/hosted_chat_entrypoint.py +++ b/src/fi/alk/harness/hosted_chat_entrypoint.py @@ -39,6 +39,7 @@ ) from .tools import schema from .session import ARTIFACT, DONE, RESULT, STAGE_IDLE_TIMEOUT_SECONDS, TEXT, TOOL, Event, Stage +from ..providers.secret_shapes import ChatHold from ..providers.agent import ( _LETTER, _QUERY_OR_FRAGMENT, @@ -167,6 +168,11 @@ # cleaned token by token, split at every whitespace; any whitespace is then a cut, and each # piece is cleaned the prose way too, so it splits at the same tokens and keeps its whitespace. # +# The secret shapes cross whitespace, though: a password broken by a space or a line break, a +# query on the next piece, a named value and its value, and a key block. So the cut is narrowed by +# ChatHold (providers/secret_shapes.py): any whitespace is a cut only where none of those is +# still open or has just closed, and the shed below asks it where it may split (TH-9500). +# # The holdback is bounded. Past _HOLD_SOFT characters with no cut, what is held is shed: # - with a ? or # after its first URL start, the rest of the token is that URL's query. The held # text is sent cleaned (the URL with its host and path), and the rest of the token is dropped @@ -191,6 +197,13 @@ class _DeltaCleaner: """The cleaned text of one streamed message, let out piece by piece.""" + # A run of the breaks the one-URL join removes (cr3 H1, `_line_cut`). + _BREAK_RUN = re.compile(r"[\t\r\n]+") + # Where a URL's authority ends, and so its userinfo (`_line_cut`). + _AUTHORITY_END = re.compile(r"[/?#]") + # A scheme's characters, any of them first (`_learn_past`, cr3 M6). + _RUN_CHARS = re.compile(r"[A-Za-z0-9+.\-]*") + def __init__(self) -> None: self.held = "" self.dropping = False @@ -202,6 +215,14 @@ def __init__(self) -> None: # Before prose is known: a space has come after the opening began, with only whitespace # after it so far. self._spaced = False + # Past the soft bound, while the opening's run may still end in something other than + # "://" (`_learn_past`). + self._past: str | None = None + # Before prose: the joined index of a cut `_line_cut` held at 0, while the held text has + # only grown since (cr4 M2). + self._held_cut: int | None = None + # Where a cut may fall without splitting a secret's match (TH-9500). + self._hold = ChatHold() def feed(self, text: str) -> str: out: list[str] = [] @@ -223,19 +244,21 @@ def feed(self, text: str) -> str: return "".join(out) text = text[cut.start() :] self.dropping = False - last = _ANY_CUT.search(text[::-1]) if self.prose else None - if last is None: - self.held += text - else: - end = len(text) - last.start() - out.append(self._clean(self.held + text[:end])) - self.held = text[end:] + # TH-9500: the seam cuts where this would, at the last whitespace once the text is + # prose, but never into a match that is still open or has just closed: rule 1, rule 2, a + # named value and a key block all cross whitespace. + self._hold.prose = self.prose + end = self._hold.cut(self.held, text) + whole = self.held + text + if end: + out.append(self._clean(whole[:end])) + self.held = whole[end:] if len(self.held) > _HOLD_SOFT: out.append(self._shed()) return "".join(out) def flush(self) -> str: - held, self.held, self.dropping = self.held, "", False + held, self.held, self.dropping, self._held_cut = self.held, "", False, None return self._clean(held) if held else "" def _learn(self, text: str) -> None: @@ -252,13 +275,38 @@ def _learn(self, text: str) -> None: return self._spaced = True if len(self._opening) >= _HOLD_SOFT: + self._learn_past(text) return - self._opening += text[: _HOLD_SOFT - len(self._opening)] + taken = _HOLD_SOFT - len(self._opening) + self._opening += text[:taken] run = _SCHEME_RUN.match(self._opening) rest = self._opening[run.end() :] if run else self._opening # Not a letter first, or a scheme's run followed by anything but "://". if run is None or (rest and not "://".startswith(rest[:3])): self.prose = True + elif len(self._opening) >= _HOLD_SOFT: + # The opening's run, or its "://", isn't over at the soft bound: what follows still + # decides, as cleaning the whole text decides it (cr3 M6). Held as one URL, a run + # that ends in a line break would have the URL lines after it glued onto it, and + # their queries read as its path. + self._past = rest + self._learn_past(text[taken:]) + + def _learn_past(self, text: str) -> None: + """Past the soft bound: the opening's run, then "://" or not (`_past` holds what + followed the run so far; "" while the run goes on, None once decided).""" + past = self._past + if past is None or not text: + return + if not past: + end = self._RUN_CHARS.match(text).end() + if end == len(text): + return + text = text[end:] + past = (past + text)[:3] + if not "://".startswith(past): + self.prose = True + self._past = None if self.prose or past == "://" else past def _clean(self, piece: str) -> str: if not self.prose: @@ -266,37 +314,157 @@ def _clean(self, piece: str) -> str: # is after the message's last character (no part of the stripped reply), so it is # kept as it came. body = piece.rstrip() - return _redact(body.translate(_URL_BREAKS)) + piece[len(body) :] + return self._joined_clean(body) + piece[len(body) :] # The space in front only keeps the cleaner from reading the piece as one URL, which # would drop its tabs and line breaks; the prose way keeps them, and the space comes off. return _redact(" " + piece)[1:] + def _line_cut(self, raw: str, joined: str, at: int) -> int: + """Before prose, where in the held text a cut the branches put at `at` in its joined form + falls: at the start of the break run before the line that holds it (cr3 H1). 0 when + nothing may go yet. + + The branches find URL starts on the joined text, where a line is glued onto the next + line's scheme (`DB_PASSWORD=V` then `https://…` reads `Vhttps://`). Cut there, the name + would go out alone and its value be cleaned as a scheme, lower-cased. Cleaning the whole + text masks line starts before the join, so a cut keeps lines whole, and each piece sent is + masked and joined as the whole is. + + Nothing is sent while 0 holds, so the held text only grows; the answer for the same `at` + stays 0 until a later URL start moves `at`. It is kept (`_held_cut`), so a long hold + isn't walked again on every delta (cr4 M2).""" + if at == self._held_cut: + return 0 + # The break runs, found once: the walk goes back over them by index, so a line costs + # its own length however far the held text reaches back. + runs = [(found.start(), found.end()) for found in self._BREAK_RUN.finditer(raw)] + end, line_at, i = len(raw), len(joined), len(runs) + while True: + # Back over the lines from the end, to the one that holds `at`. + i -= 1 + if i < 0: + self._held_cut = at + return 0 + run, brk = runs[i] + line_at -= end - brk + if at >= line_at: + break + end = run + # A name that begins the line before, with its value on this line (`token:` then `V`), is + # one match: the two lines stay together. Such a value is a span found with the line + # before and not without it. + if i: + from ..providers.secret_shapes import line_starts + + before = runs[i - 1][1] + stop = runs[i + 1][0] if i + 1 < len(runs) else len(raw) + own = {span[0] + brk - 1 for span in line_starts(raw[brk - 1 : stop])} + with_before = line_starts(raw[before - 1 : stop]) + if any( + span[0] + before - 1 >= brk and span[0] + before - 1 not in own + for span in with_before + ): + line_at -= run - before + run = runs[i - 1][0] + # A URL before the cut keeps its userinfo with its host: the text after the cut, up to + # the next ://, may still be that userinfo (`postgres://admin:Xk9/Qp+Zr`, then `8@db…`, + # a rule 1b password holding a /). So while an @ comes before that :// (the last URL's + # own, which the cut is at or before), nothing goes. Nor does it while the text before + # the cut ends in a userinfo still open: an @ after a : since the last ://, with no /, + # ? or # between that @ and the cut (`…:P@ss/Xk9@`, then the host). The last :// is + # looked for in the joined text, where a scheme broken by a line break is whole again. + scheme = joined.rfind("://", 0, line_at) + # A cut at the start of a line that is only a URL's own `://` (its scheme on the line + # before) would send the scheme alone and leave the userinfo to be cleaned without it, + # so nothing goes. A known limit: an honest reply with such a line and then more than + # about 12 K of glued URLs reaches the hard bound and streams `[redacted]`. + inside = joined.rfind("://", 0, line_at + 3) + if inside >= 0 and line_at - 3 < inside <= line_at: + self._held_cut = at + return 0 + if scheme >= 0: + following = joined.find("://", line_at) + sign = joined.rfind("@", scheme, line_at) + if "@" in joined[line_at:following] or ( + sign > joined.find(":", scheme + 3, sign) >= 0 + and not self._AUTHORITY_END.search(joined, sign, line_at) + ): + self._held_cut = at + return 0 + return run + + def _joined_clean(self, text: str) -> str: + """Held text before prose, cleaned: masked at its line starts, then joined, as the whole + text is (cr2 H1). The kinds masked go on this clean's one log line.""" + from ..providers.agent import _line_starts_joined + + found: list[str] = [] + joined = _line_starts_joined(text, found) + return _redact(joined, hits=found) + def _shed(self) -> str: - held = self.held + # 1. The limit: feed calls this once more than _HOLD_SOFT characters are held. + # 2. The split, from the seam (TH-9500): None when nothing it holds is open, so the + # branches below run on all of the held text; otherwise the start of the held text's + # first open match that may not be cut: a key block, or a key, whose hold starts at + # its URL piece when it is in one. + split = self._hold.shed_cut(self.held) + # 6. A limit of 0: the held text opens with such a match, so nothing goes out yet. + if split == 0: + return "" + rest = "" + if split is not None: + self.held, rest = self.held[:split], self.held[split:] + # 3. The branches, run on the text before the split. Before prose they run on the + # joined text, and a cut keeps lines whole (`_line_cut`, cr3 H1). + held = head = raw = self.held if not self.prose: if self._spaced: - self.held = "" + self.held, self._held_cut = "", None return self._clean(held) - self.held = held = held.translate(_URL_BREAKS) + held = raw.translate(_URL_BREAKS) starts = [] - for found in _START.finditer(held): - letter = _LETTER.search(held, found.start(), found.end() - 3) - starts.append(letter.start() if letter else found.end() - 3) + for start in _START.finditer(held): + letter = _LETTER.search(held, start.start(), start.end() - 3) + starts.append(letter.start() if letter else start.end() - 3) + + def cut(at: int) -> tuple[str, str]: + # The held text before `at` (an index into `held`), cleaned, and the rest, held again. + if self.prose: + return _redact(held[:at]), held[at:] + if len(raw) != len(held): + at = self._line_cut(raw, held, at) + if not at: + return "", raw + # Text goes, so the held text's indexes move: a held cut no longer stands. + self._held_cut = None + return self._joined_clean(raw[:at]), raw[at:] + if not starts: tail = _SCHEME_TAIL.search(held) - keep = tail.start() if tail else len(held) - self.held = held[keep:] - return _redact(held[:keep]) - if _QUERY_OR_FRAGMENT.search(held, starts[0]): - self.held, self.dropping = "", True - return _redact(held) - out = "" - if starts[-1] > 0: - out = _redact(held[: starts[-1]]) - self.held = held = held[starts[-1] :] - if len(held) > _HOLD_HARD: - self.held, self.dropping = "", True - out += _REDACTED + out, self.held = cut(tail.start() if tail else len(held)) + elif _QUERY_OR_FRAGMENT.search(held, starts[0]): + out = _redact(held) if self.prose else self._joined_clean(raw) + self.held, self.dropping, self._held_cut = "", True, None + else: + out, self.held = "", raw + if starts[-1] > 0: + out, self.held = cut(starts[-1]) + # The held text itself is counted, breaks too: the join drops them, but they are held, + # and the walk over them (`_line_cut`) is bounded by this. + if len(self.held) > _HOLD_HARD: + self.held, self.dropping, self._held_cut = "", True, None + out += _REDACTED + # 5. The drop rule. A key inside a held URL piece is never a cut point: the split above + # is that piece's start or nothing. A split is a piece's start, so the token a branch + # dropped ends before it, and no fragment of it is kept. When the text the branches + # ran on ends in whitespace its last token is complete, so nothing is left to drop + # and the next honest word is kept (r1 L6). Before prose a tab or a line break is no + # token's end: the one-URL join removes it. + if self.prose and self.dropping and head[-1:].isspace(): + self.dropping = False + # 4. The tail from the split is held again. + self.held += rest return out diff --git a/src/fi/alk/harness/outbound.py b/src/fi/alk/harness/outbound.py index bac09d384..0a08438c4 100644 --- a/src/fi/alk/harness/outbound.py +++ b/src/fi/alk/harness/outbound.py @@ -70,6 +70,7 @@ ) from fi.simulate.results.futureagi import MAX_TARGET_LATENCY_TURNS +from ..providers.agent import _log_redacted, _redact_text from .job import FailureDomain, HarnessStage logger = logging.getLogger(__name__) @@ -277,28 +278,37 @@ def _mask_userinfo(match: re.Match[str]) -> str: def redact_outbound_text(value: str, extra_secret_values: tuple[str, ...] = ()) -> str: """Scrubs a single free-text field before it can leave the sandbox on any of the three channels (outbound-channels.md v1.3 "Redaction (enforced before emit)"; hosted-execution- - seams.md v1.11 §3 "any outbound projection ... redacts userinfo"). Two things, applied in - order: - - 1. URL userinfo (`_USERINFO_PATTERN`/`_mask_userinfo`, above) -- a password is masked and the - username kept (matching the contract's own `postgresql://harness:***@...` example); a bare - token/username-only userinfo (no `:`) is masked outright, since that shape is how a bearer - token appears, not a username. - 2. `extra_secret_values` -- exact-substring replacement for a caller-supplied list of secret - values. Always `()` today: the adapter (P10) is what will know the job's declared secrets - and pass them in -- this parameter exists now so `build_event_record`/`build_result_receipt` - never need to change shape when that wiring lands. - - NOT a general secret-content scanner -- the contract's "same secret-content scan as the - artifact sealer" is a separate, sealer-side obligation. This is the narrow subset this module - can enforce on every string field it controls without false-positiving on ordinary diagnostic - text. + seams.md v1.11 §3 "any outbound projection ... redacts userinfo"). Three things, applied in + order (TH-9500): + + 1. `extra_secret_values` -- exact-substring replacement for a caller-supplied list of secret + values. First, so a declared secret is matched whole before the cleaner below can mask + part of it and leave the rest unmatched. Always `()` today: the adapter (P10) is what will + know the job's declared secrets and pass them in -- this parameter exists now so + `build_event_record`/`build_result_receipt` never need to change shape when that wiring + lands. + 2. The kit's one cleaner, `providers.agent._redact_text`, all three of its steps: a URL loses + its userinfo, query and fragment, and keys, key blocks and named values are masked by + their shape. It logs nothing itself; one line names what it removed, on the cleaner's + own logger (not one of the loggers `MaskSecrets` filters, so the filter never re-enters), + and only when it changed the text. + 3. URL userinfo (`_USERINFO_PATTERN`/`_mask_userinfo`, above), for whatever step 2 left: a + password is masked and the username kept; a bare token/username-only userinfo (no `:`) + is masked outright, since that shape is how a bearer token appears, not a username. + + Step 2 removes a URL's userinfo outright, so `https://user:pw@host` now comes out as + `https://host`, not `https://user:***@host`. """ - redacted = _USERINFO_PATTERN.sub(_mask_userinfo, value) + redacted = value for secret in extra_secret_values: if len(secret) >= _SHORTEST_REDACTABLE_SECRET: redacted = redacted.replace(secret, "***") - return redacted + refused: list[str] = [] + hits: list[str] = [] + cleaned = _redact_text(redacted, refused, hits) + if (refused or hits) and cleaned != redacted: + _log_redacted(refused, hits) + return _USERINFO_PATTERN.sub(_mask_userinfo, cleaned) # ================================================================================================= diff --git a/src/fi/alk/harness/scenario_source.py b/src/fi/alk/harness/scenario_source.py index b4398a473..63361edb2 100644 --- a/src/fi/alk/harness/scenario_source.py +++ b/src/fi/alk/harness/scenario_source.py @@ -276,6 +276,12 @@ class _CompiledScenario: # same document and sent at pre-allocation, so a call can be read on the platform without the # scenario file beside it. Presentation only: nothing in the scheduler looks at it. presented: dict[str, Any] = field(default_factory=dict) + # The fields `presented` is built from, uncut (`_presented(..., fields=True)`). The provision + # payload cleans them and builds its record from them (TH-9500 cr2 M1, cr3 L4): `persona_of` + # cuts `name`, `role` and `scenario_name` at 255 characters and rewrites the name it shows, + # and a key cut or rewritten there no longer has a key's shape. Kept uncleaned, so loading a + # scenario cleans and logs nothing; `presented` stays as written, since the judge reads it. + presented_fields: dict[str, Any] = field(default_factory=dict) source_scenario_key: str | None = None # One correct way through the call: handed to reference checks and shown to the judge. solution: tuple[dict[str, Any], ...] = () @@ -313,12 +319,18 @@ def _validate_subgoal_name(name: str, *, folder_name: str) -> None: ) -def _presented(body: dict[str, Any], *, scenario_key: str) -> dict[str, Any]: +def _presented( + body: dict[str, Any], *, scenario_key: str, fields: bool = False +) -> dict[str, Any]: """The persona record the platform stores for a scenario, built by the platform module's own helper rather than a second copy of it here, so both paths describe a person the same way. Presentation only, and never load-bearing: anything missing from the document is simply absent from what the platform shows, and a document this cannot read still runs. + + With `fields`, the fields `persona_of` would read, `scenario_key` among them, uncut and as + written. They are the same keys `body` has, so a record built from them, cleaned, is + `_presented(cleaned, scenario_key=...)` (TH-9500 cr3 L4). """ from types import SimpleNamespace @@ -334,17 +346,17 @@ def _presented(body: dict[str, Any], *, scenario_key: str) -> dict[str, Any]: use_case = scenario_key.replace("-", " ").replace("_", " ") use_case = use_case[:1].upper() + use_case[1:] + read = { + "persona": body.get("persona") or {}, + "name": str(body.get("name") or ""), + "instruction": str(body.get("instruction") or ""), + "tests": str(body.get("tests") or ""), + "use_case": use_case, + } + if fields: + return {**read, "scenario_key": scenario_key} try: - return persona_of( - SimpleNamespace( - persona=body.get("persona") or {}, - name=str(body.get("name") or ""), - scenario_key=scenario_key, - instruction=str(body.get("instruction") or ""), - tests=str(body.get("tests") or ""), - use_case=use_case, - ) - ) + return persona_of(SimpleNamespace(**read, scenario_key=scenario_key)) except Exception: # noqa: BLE001 - a scenario never fails to run over how it is displayed return {} @@ -558,6 +570,7 @@ def _load_one( setup=setup, ready=ready, presented=_presented(body, scenario_key=scenario_key), + presented_fields=_presented(body, scenario_key=scenario_key, fields=True), solution=tuple(solution), ) @@ -935,14 +948,29 @@ def _provision_payload( `scenario.json` for scheduler-facing fields only, and the cost was a platform that could show a call but not who was on it or what they came for. """ + + def cleaned(scenario: Any) -> dict[str, Any]: + # `persona_of` over the cleaned fields (cr2 M1), here at the send, so its log line is + # the send's (cr3 L4). Cleaned again after: the shown name's `-` and `_` become spaces, + # which can give a key back its shape (`call_AKIA…` shows as `Call AKIA…`, cr3 M4). + read = dict(scenario.presented_fields) + if not read: + # No document was read (or it couldn't be): send the bare key, never a record + # `persona_of` would make up from nothing ("caller", "Scenario"). + return {} + scenario_key = str(read.pop("scenario_key", "") or "") + return _redact(_presented(_redact(read), scenario_key=scenario_key)) + payload: dict[str, Any] = { "operation": "provision", "name": run_name, # Everything the serializer accepts, where the document had it: name, role, situation, # outcome and the persona itself. `scenario_key` is set last because it is the one field the # platform matches on and it must be this scenario's, whatever the persona record says. + # The record is cleaned first (TH-9500): it is free text from the customer's own files, + # and this is the one place it leaves for the platform. "personas": [ - {**scenario.presented, "scenario_key": scenario.scenario_key} + {**cleaned(scenario), "scenario_key": scenario.scenario_key} for scenario in scenarios ], } diff --git a/src/fi/alk/harness/session.py b/src/fi/alk/harness/session.py index 739f04ba1..ffb364dee 100644 --- a/src/fi/alk/harness/session.py +++ b/src/fi/alk/harness/session.py @@ -25,6 +25,7 @@ from dataclasses import dataclass, field from typing import Any +from ..providers.agent import _redact from . import spend from .live import Channel, carrying, folded_in from .backends import ( @@ -385,7 +386,7 @@ def _events(self, received: Any, turn: Turn) -> list[Event]: events = [ Event( RESULT, - text=_shown(received.text), + text=_shown(_redact(received.text)), detail={ "is_error": received.is_error, "call_id": received.id, diff --git a/src/fi/alk/providers/agent.py b/src/fi/alk/providers/agent.py index 753df1e0b..cdffe8054 100644 --- a/src/fi/alk/providers/agent.py +++ b/src/fi/alk/providers/agent.py @@ -8,6 +8,7 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator +from . import secret_shapes from .setup_hash import compute_setup_hash logger = logging.getLogger(__name__) @@ -244,16 +245,29 @@ def _authored_model(value: Any) -> dict[str, Any] | None: return {"provider": provider, "name": name, "temperature": None, "max_tokens": None} -def _redact(value: Any, key: str = "") -> Any: +def _redact(value: Any, key: str = "", hits: list[str] | None = None) -> Any: + """`value` cleaned, with one log line when anything was masked. `hits` are kinds the caller + already masked in `value` before handing it here (chat's line-start mask, `_line_starts_joined`), + so they go on the same line.""" refused: list[str] = [] - redacted = _redact_in(value, key, refused) - if refused: - # Once per document, not per string: a crafted contract can hold half a million of them. - logger.warning("a URL was redacted whole: class=%s", refused[0]) + found = list(hits or ()) + redacted = _redact_in(value, key, refused, found) + if (refused or found) and (redacted != value or hits): + # Once per call, not per string: a crafted contract can hold half a million of them. + _log_redacted(refused, found) return redacted -def _redact_in(value: Any, key: str, refused: list[str]) -> Any: +def _log_redacted(refused: list[str], hits: list[str]) -> None: + """The cleaner's one line: the URL class it refused and the kinds of secret it masked, never + the text.""" + parts = [f"url={refused[0]}"] if refused else [] + if hits: + parts.append("kinds=" + ",".join(sorted(set(hits)))) + logger.warning("secrets redacted: %s", "; ".join(parts)) + + +def _redact_in(value: Any, key: str, refused: list[str], hits: list[str]) -> Any: lowered = key.lower() if any(marker in lowered for marker in _REDACTION_MARKERS): return _REDACTED @@ -263,13 +277,13 @@ def _redact_in(value: Any, key: str, refused: list[str]) -> Any: return _REDACTED if isinstance(value, Mapping): return { - str(item_key): _redact_in(item, str(item_key), refused) + str(item_key): _redact_in(item, str(item_key), refused, hits) for item_key, item in value.items() } if isinstance(value, list): - return [_redact_in(item, key, refused) for item in value] + return [_redact_in(item, key, refused, hits) for item in value] if isinstance(value, str): - return _redact_text(value, refused) + return _redact_text(value, refused, hits) return value @@ -285,27 +299,75 @@ def _redact_in(value: Any, key: str, refused: list[str]) -> Any: _QUERY_OR_FRAGMENT = re.compile(r"[?#]") -def _redact_text(value: str, refused: list[str]) -> str: - """Drop every URL's userinfo, query and fragment, and keep the rest of the text as written. +def _redact_text(value: str, refused: list[str], hits: list[str] | None = None) -> str: + """Clean one string in three steps (TH-9500), and keep the rest of the text as written. + + 1. The leftover rules, on the raw value: a password broken by whitespace, ?, # or /, a + userinfo holding an @ before the password's / (rule 1b), and a query after a break + (`secret_shapes.leftovers`). A value that is one URL is joined first (`_one_url`). + 2. TH-9491's URL step: drop every URL's userinfo, query and fragment. A URL runs from its + scheme to the next whitespace. Within one whitespace-free token, a later :// that starts + before the current URL's own ? or # begins a new URL; once a URL has a ? or #, the rest + of the token is its query or fragment. A key in a URL's host makes the whole URL + [redacted], matched before urlsplit lowers the host. + 3. The key patterns, on step 2's output (`secret_shapes.find`). - A URL runs from its scheme to the next whitespace. Within one whitespace-free token, a later - :// that starts before the current URL's own ? or # begins a new URL; once a URL has a ? or - #, the rest of the token is its query or fragment. + The kinds masked are added to `hits`; the caller logs them, so this logs nothing. """ - if "://" not in value: - return value - if _LEADING_SCHEME.match(value): - joined = value.translate(_URL_BREAKS) - if not _SPACE.search(joined): - # A field that is one URL, perhaps broken by a tab or a line break: one token. - return _redact_token(joined, refused) + # Both 600-character tests are made on the raw value: step 2 can shorten it (r1 M1). + cut = secret_shapes.is_cut(value) + joined = _one_url(value, hits) + if joined is not None: + # A value that is one URL runs step 1 on its joined token, as chat's `_clean` does, so + # the whole text agrees with the stream and rule 1b runs once (addendum M3, M-A): on the + # raw value, rule 1 would take a password broken by a line break up to its first @. + value = secret_shapes.leftovers(joined, hits) + value = _redact_token(value, refused, hits, secret_shapes.may_hold_key(value)) + return secret_shapes.mask(value, secret_shapes.find(value, cut), hits) + value = secret_shapes.leftovers(value, hits) + if "://" in value: + value = _redact_urls(value, refused, hits, secret_shapes.may_hold_key(value)) + return secret_shapes.mask(value, secret_shapes.find(value, cut), hits) + + +def _one_url(value: str, hits: list[str] | None) -> str | None: + """The one token a field that is one URL, perhaps broken by a tab or a line break, is + cleaned as; None when it is not one URL.""" + if not _LEADING_SCHEME.match(value): + return None + if _SPACE.search(value.translate(_URL_BREAKS)): + return None + return _line_starts_joined(value, hits) + + +def _line_starts_joined(value: str, hits: list[str] | None) -> str: + """`value` with its tabs and line breaks removed, as a URL joins, after the keys and named + values that begin a line or follow a tab are masked. Once joined, such a key would follow the + URL's last character, where no pattern starts (r2 M1). Chat's text before prose is joined here + too, so its stream masks what the whole text masks (change review r2, cr2 H1).""" + starts = secret_shapes.line_starts(value) + if starts: + # A line's first word is never inside a URL's host. + value = secret_shapes.mask(value, starts, hits, widen=False) + return value.translate(_URL_BREAKS) + + +def _redact_urls( + value: str, refused: list[str], hits: list[str] | None, keys: bool +) -> str: + # Step 1 can leave one URL where the raw value had a space: rule 1's password broken by one. + joined = _one_url(value, hits) + if joined is not None: + return _redact_token(joined, refused, hits, keys) return "".join( - part if not part or part.isspace() else _redact_token(part, refused) + part if not part or part.isspace() else _redact_token(part, refused, hits, keys) for part in _SPACE.split(value) ) -def _redact_token(token: str, refused: list[str]) -> str: +def _redact_token( + token: str, refused: list[str], hits: list[str] | None = None, keys: bool = False +) -> str: starts: list[tuple[int, bool]] = [] for match in _START.finditer(token): # The scheme is the run from its first letter; the text before that letter stays. @@ -326,7 +388,14 @@ def _redact_token(token: str, refused: list[str]) -> str: following = starts[index + 1][0] if index + 1 < len(starts) else len(token) end = len(token) if query_at < following else following piece = token[start:end] - if schemeless: + kind = secret_shapes.host_key(piece) if keys else None + if kind is not None: + # A key in the host: the whole URL goes, so step 3 never writes inside a host and a + # second pass changes nothing (r2 L3). + out.append(_REDACTED) + if hits is not None: + hits.append(kind) + elif schemeless: # "${SCHEME}://user:pw@host": parsed with a stand-in scheme, written back without it. cleaned = _redact_url("x" + piece, refused) out.append(cleaned[1:] if cleaned.startswith("x://") else cleaned) diff --git a/src/fi/alk/providers/secret_shapes.py b/src/fi/alk/providers/secret_shapes.py new file mode 100644 index 000000000..a923ceb98 --- /dev/null +++ b/src/fi/alk/providers/secret_shapes.py @@ -0,0 +1,1359 @@ +"""Secrets found by their shape, for the kit's one cleaner (TH-9500, design page v2.4). + +`providers.agent._redact_text` cleans a string in three steps: the two leftover rules here, on +the raw value; TH-9491's URL step; then the key patterns here (`find`), on that step's output. +Hosted chat sends its reply in pieces through the same cleaner, and `ChatHold` says where a +piece may end, so no rule here is split between two pieces. + +Nothing here does I/O, and nothing is imported but `re` and `bisect`. The kit's own pattern list +(`fi/evals/guardrails/scanners/secrets.py`) costs about 0.25 s to import and belongs to the evals +package, so only the drift test reads it, to tie the prefix patterns here to it. +""" + +import re +from bisect import bisect_left + +REDACTED = "[redacted]" + +# A key's own characters. A literal that follows one of them is inside a longer run, so no +# pattern starts there: each run is tried once, which keeps every scan linear. +_KEY = r"A-Za-z0-9_\-" +# The 600-character cut (`session._shown`) ends a text with this. +_AT_CUT = r"(?=\.\.\.\Z)" +_CUT_LENGTH = 600 + + +def _start(literal: str) -> str: + escaped = re.escape(literal) + # The literal comes first, so the regex engine's literal search finds each start. + return f"{escaped}(? str: + # A letter and a digit, or 16 or more characters. Read from the value's start and bounded + # at 16, so a name whose value fails is dropped by the engine, not by a Python check. + def test(char: str) -> str: + return rf"(?:(?={char}{{16}})|(?={char}{{0,15}}?{letter})(?={char}{{0,15}}?\d))" + + double, single, back = r'[^"\n]', r"[^'\n]", r"[^`\n]" + # A value already masked fails at once, so a second pass costs little. + return ( + r"(?![\"'`]?\[redacted\])" + r'(?:(?P")' + + test(double) + + r"|(?P')" + + test(single) + + r"|(?P`)" + + test(back) + + "|" + + test(_PLAIN_VALUE) + + ")" + ) + + +def _per_form(start: str) -> dict[str, re.Pattern[str]]: + # One regex per name, on the lower-cased text, each opening with its name, so the engine's + # literal search finds it as str.find would: a run of a name's own first letter never + # starts an attempt. `start` is a lookbehind on the name that says where it may sit. + tail = r"(?!::)" + _TAIL + _value_start("[a-z]") + return { + form: re.compile(re.escape(form) + start.format(re.escape(form)) + tail) + for form in _FORMS + } + + +# A name followed by "::" is code, not a name (Token::new(42), r1 L2). +# Not right after a letter or digit: the whole identifier, or its end after "_" or "-". "pass" +# only ever follows "_" or "-". +_NAMED = _per_form(r"(? bool: + """False when `text` holds no key literal at all, so no URL's host needs checking.""" + return any(root in text for root in _LITERAL_ROOTS) + + +_HEADER = "-----BEGIN " + + +# ---------------------------------------------------------------------------------- step 1 +# Rule 1: from scheme://user: to the next @, when the password is broken by whitespace, ?, # +# or / (TH-9491's URL step stops a URL at each). Its userinfo is removed when the text after +# "user:" doesn't begin with a port, holds at most one run of whitespace with text on both +# sides, and is at most 64 characters on each side of that run. +_PORT = ( + r"(?:\d+|\$?\{[^{}\s]*\}|\$[A-Za-z_][A-Za-z0-9_]*|%[A-Za-z_][A-Za-z0-9_]*%|<[^<>\s]*>" + r"|[A-Z][A-Z0-9_]*|[Pp][Oo][Rr][Tt])(?=[^A-Za-z0-9_\-@]|\Z)" +) +_HALF = r"[^\s@]{1,64}" +# A line break, after up to 4 spaces or tabs, with up to 16 characters of indent; or up to 4 +# spaces, tabs or NBSPs (r3 H2, r4 L3). +_RUN = r"(?:[ \t]{0,4}\r?\n[ \t]{0,16}|[ \t\xa0]{1,4})" +_SCHEME = r"(?\]}|]|\Z))" +) +_RULE1B = re.compile( + _SCHEME + # The port test (M1): with no : before the authority's first @, the text after its last : + # must not read as a port, so admin@localhost:8080/bob@example.com keeps its host. + + rf"(?![^\s/?#:@]{{0,129}}@[^\s/?#]{{0,194}}:{_PORT}/)" + + r"(" + # The authority holds a :, and it comes before any [ (M-C): admin@[::1]/bob@x keeps its host. + + r"(?=[^\s/?#\[]{0,129}:)" + # At most 64 characters before its first : or @. + + r"(?=[^\s/?#:]{0,64}[:@])" + # Up to the authority's last @, then at most 64 to the /, then 1-64 to the path's @. + + r"[^\s/?#\[\]]{0,129}@[^\s/?#@]{0,64}/[^\s/?#@]{1,64}@" + + r")" + + rf"(?={_RULE1B_HOST})" +) + +# Rule 2: the piece after an address piece loses its query values when it starts with ?, & or +# #, or whatever it starts with when the address's query ends in & or ?. +# The lookahead finds the piece's :// once (a lookahead never retries), so a long piece with +# many :// is scanned a bounded number of times. +_THEN_PIECE = re.compile(r"(? list[tuple[int, int]]: + """The userinfo rules 1 and 1b remove from `text`: each span runs from the user to its @. + + One sorted list, with spans that overlap or touch merged, since `remove` needs them in order + and apart (addendum M4): a://u:x://v:p@h/x@host.example is both rules' at once.""" + if "://" not in text or "@" not in text: + return [] + spans = [match.span(1) for match in _RULE1.finditer(text)] + rule1b(text) + spans.sort() + merged: list[tuple[int, int]] = [] + for start, end in spans: + if merged and start <= merged[-1][1]: + merged[-1] = (merged[-1][0], max(merged[-1][1], end)) + else: + merged.append((start, end)) + return merged + + +def rule1b(text: str) -> list[tuple[int, int]]: + """The userinfo rule 1b removes from `text`, up to the path's @. Its own function, so the + speed test can time the cleaner without it.""" + return [match.span(1) for match in _RULE1B.finditer(text)] + + +def rule2(text: str) -> list[tuple[int, int]]: + """The query values rule 2 empties in `text`: the piece after an address piece.""" + # It needs a ?, & or # somewhere: a query ends in one, or the next piece starts with one. + if "://" not in text or ("?" not in text and "&" not in text and "#" not in text): + return [] + spans: list[tuple[int, int]] = [] + for match in _THEN_PIECE.finditer(text): + address = match.group(1) + if ( + address[-1] in "&?" + and "?" not in address + and text[match.end()] not in "?&#" + ): + continue + piece = _PIECE.match(text, match.end()) + start = piece.start() + for param in _PARAM.finditer(piece.group()): + lead, name, value = param.groups() + # An empty value or the marker never matches, so cleaning twice changes nothing. + if not value or value == REDACTED: + continue + if lead == "&" and name.split(";")[0] in _ENTITIES: + continue + spans.append((start + param.start(3), start + param.end(3))) + return spans + + +def remove(text: str, spans: list[tuple[int, int]]) -> str: + """`text` without `spans`, which are in order and don't overlap.""" + out, at = [], 0 + for start, end in spans: + out.append(text[at:start]) + at = end + out.append(text[at:]) + return "".join(out) + + +def leftovers(value: str, hits: list[str] | None = None) -> str: + """Step 1: rule 1, then rule 2 on what rule 1 leaves.""" + spans = rule1(value) + if spans: + value = remove(value, spans) + if hits is not None: + hits.append("rule1") + spans = rule2(value) + if spans: + value = remove(value, spans) + if hits is not None: + hits.append("rule2") + return value + + +# ---------------------------------------------------------------------------------- step 3 +def is_cut(value: str) -> bool: + """Whether `value` is a text cut at 600 characters: both the length and the "..." (r4 L1). + Tested on the raw value, before step 1, so a URL query that step 2 drops before a key body + can't move it (r1 M1).""" + return len(value) == _CUT_LENGTH and value.endswith("...") + + +def find(text: str, cut: bool = False) -> list[tuple[int, int, str]]: + """Step 3: what to mask in `text`, as (start, end, kind). + + Keys by prefix or shape, key blocks, tokens in a URL's path and named values. With `cut`, + the raw value was a 600-character cut, so a key block with no END runs to the end and a key + cut short at the end is masked to it. + + Literals and names are looked for with `in` first (names in the lower-cased text), so text + with neither runs no pattern regex. + """ + low = text.translate(_ASCII_LOWER) + names = any(root in low for root in _NAME_ROOTS) + if not names and not any(root in text for root in _LITERAL_ROOTS): + return [] + literals = [literal for literal in _KEY_LITERALS if literal in text] + paths = [path for path in _PATHS if path[1] in text] + header = _HEADER in text + telegram = ":AA" in text + jwt = "eyJ" in text + guards = [guard for guard in _GUARDS if guard[1] in low] if names else [] + spans: list[tuple[int, int, str]] = [] + for kind, literal, pattern, cut_pattern in _PREFIXES: + if literal in literals: + spans.extend((m.start(), m.end(), kind) for m in pattern.finditer(text)) + if cut: + spans.extend( + (m.start(), m.end(), "cut_" + kind) + for m in cut_pattern.finditer(text) + ) + for kind, _, pattern, cut_pattern in paths: + spans.extend((m.start(1), m.end(1), kind) for m in pattern.finditer(text)) + if cut: + spans.extend( + (m.start(), m.end(), "cut_" + kind) for m in cut_pattern.finditer(text) + ) + if telegram: + _telegram(text, _TELEGRAM, "telegram", spans) + if cut: + _telegram(text, _TELEGRAM_CUT, "cut_telegram", spans) + if jwt: + spans.extend((m.start(), m.end(), "jwt") for m in _JWT.finditer(text)) + for kind, _, pattern in guards: + spans.extend((m.start(1), m.end(1), kind) for m in pattern.finditer(low)) + if header: + spans.extend((start, end, "private_key") for start, end in _blocks(text, cut)) + if names: + _named(text, low, spans) + return spans + + +def line_starts(text: str) -> list[tuple[int, int, str]]: + """Keys and named values that begin a line or follow a tab in `text`. + + For a value TH-9491 cleans as one URL: it joins the value's lines, which puts each line's + first word right after the URL's last character, where no pattern starts (r2 M1, r3 M7). + """ + spans: list[tuple[int, int, str]] = [] + for kind, literal, pattern, _ in _PREFIXES: + if literal in text: + spans.extend( + (m.start(), m.end(), kind) + for m in pattern.finditer(text) + if m.start() and text[m.start() - 1] in "\n\r\t" + ) + if ":AA" in text: + found: list[tuple[int, int, str]] = [] + _telegram(text, _TELEGRAM, "telegram", found) + spans.extend( + span for span in found if span[0] and text[span[0] - 1] in "\n\r\t" + ) + if "eyJ" in text: + spans.extend( + (m.start(), m.end(), "jwt") + for m in _JWT.finditer(text) + if m.start() and text[m.start() - 1] in "\n\r\t" + ) + low = text.translate(_ASCII_LOWER) + if any(root in low for root in _NAME_ROOTS): + _named(text, low, spans, at_line=True) + return spans + + +def mask( + text: str, + spans: list[tuple[int, int, str]], + hits: list[str] | None = None, + widen: bool = True, +) -> str: + """`text` with each span replaced by the marker, overlapping spans as one. + + A span that touches a URL's host widens to that whole URL: step 3 never writes inside a + host, so a second pass changes nothing (r2 L3).""" + if not spans: + return text + if widen and "://" in text: + widened = [] + for start, end, kind in spans: + url = _url_around(text, start, end) + widened.append((url[0], url[1], kind) if url else (start, end, kind)) + spans = widened + spans.sort() + out: list[str] = [] + kinds: set[str] = set() + at, begin, finish, group = 0, -1, -1, [] + for start, end, kind in spans + [(len(text) + 1, 0, "")]: + if start < finish: + # Overlapping spans are masked as one. + finish = max(finish, end) + group.append(kind) + continue + if begin >= 0: + if finish - begin != len(REDACTED) or text[begin:finish] != REDACTED: + kinds.update(group) + out.append(text[at:begin]) + out.append(REDACTED) + at = finish + begin, finish, group = start, end, [kind] + out.append(text[at:]) + if hits is not None: + hits.extend(sorted(kinds)) + return "".join(out) + + +def host_key(url: str) -> str | None: + """The kind of a key in `url`'s host, or None. Matched on the URL as written, before + urlsplit lowers the host, so an upper-case key is still found (r3 M7).""" + found = _NETLOC.search(url) + if found is None: + return None + host = found.group(1).rpartition("@")[2] + if not _HOST_LITERAL.search(host): + return None + for kind, literal, pattern, _ in _PREFIXES: + if literal in host and pattern.search(host): + return kind + if ":AA" in host: + found_tokens: list[tuple[int, int, str]] = [] + _telegram(host, _TELEGRAM, "telegram", found_tokens) + if found_tokens: + return "telegram" + if "eyJ" in host and _JWT.search(host): + return "jwt" + return None + + +_NETLOC = re.compile(r"://([^/?#]*)") +_HOST_STOP = re.compile(r"[/?#\s]") +_SCHEME_RUN_BEFORE = re.compile(r"[A-Za-z0-9+.\-]*\Z") +# A URL runs to the next whitespace or the next URL's ://, whichever comes first. +_URL_END = re.compile(r"\s|://") +# A host is looked for this far back from a match, so the search is bounded. +_HOST_REACH = 256 +# A name in a URL's host followed by a port, as in http://auth-token:8080/v1 (cr1 M1). +_HOST_PORT = re.compile(r":\d+(?=[/?#\s]|\Z)") + + +def _host_start(text: str, start: int) -> int: + """Where the host that `start` sits in begins, after its ://, or -1 when `start` is in no + URL's host. Looked for at most _HOST_REACH characters back.""" + scheme_end = text.rfind("://", max(0, start - _HOST_REACH), start) + if scheme_end < 0 or _HOST_STOP.search(text, scheme_end + 3, start): + return -1 + return scheme_end + 3 + + +def _url_around(text: str, start: int, end: int) -> tuple[int, int] | None: + """The span of the URL whose host [start, end) touches, or None.""" + host = _host_start(text, start) + if host < 0: + return None + scheme_end = host - 3 + stop = _HOST_STOP.search(text, start) + host_end = stop.start() if stop else len(text) + at = text.rfind("@", host, host_end) + if at >= 0: + host = at + 1 + if end <= host or start >= host_end: + return None + scheme = _SCHEME_RUN_BEFORE.search(text, max(0, scheme_end - 64), scheme_end) + begin = scheme.start() if scheme else scheme_end + # One search forward for the URL's end, and the next scheme's run looked for at most 64 + # back from its ://, as above: searched from every span instead, both were quadratic in a + # long run with no whitespace (cr1 H1). + stop = _URL_END.search(text, host_end) + finish = stop.start() if stop else len(text) + if stop and stop.group() == "://": + scheme = _SCHEME_RUN_BEFORE.search(text, max(host_end, finish - 64), finish) + finish = scheme.start() if scheme else finish + return begin, finish + + +def _telegram( + text: str, pattern: re.Pattern[str], kind: str, spans: list[tuple[int, int, str]] +) -> None: + for match in pattern.finditer(text): + digits = _BOT_ID.search(text[max(0, match.start() - 13) : match.start()]) + if digits: + spans.append((match.start() - len(digits.group(1)), match.end(), kind)) + + +def _blocks(text: str, cut: bool) -> list[tuple[int, int]]: + spans: list[tuple[int, int]] = [] + headers = list(_PEM.finditer(text)) + end_at, end = -2, None + done = 0 + for index, header in enumerate(headers): + if header.start() < done: + continue + # The next END is found once and reused for every header before it (r3 M6). + if end_at != -1 and end_at < header.end(): + end = _PEM_END.search(text, header.end()) + end_at = end.start() if end else -1 + following = ( + headers[index + 1].start() if index + 1 < len(headers) else len(text) + 1 + ) + if end is not None and header.end() <= end_at < following: + stop = end.end() + elif cut: + # Cut at 600 with no END: the block runs to the end of the text (r4 M6). + stop = len(text) + else: + stop = _key_lines_end(text, header.end()) + spans.append((header.start(), stop)) + done = stop + return spans + + +def _key_lines_end(text: str, start: int) -> int: + """Where a block with no END stops: after the key lines that follow its header, at the + first line that isn't one (r3 M5).""" + stop = start + rest = text[start : start + 1] + line = start if rest in ("", "\n", "\r") else _key_line_end(text, start) + if line < 0: + return stop + stop = line + newline = text.find("\n", line) + while newline >= 0: + line = _key_line_end(text, newline + 1) + if line < 0: + break + stop = line + newline = text.find("\n", line) if line < len(text) else -1 + if newline > line + 1: + # Something other than a \r came before the line break. + break + return stop + + +def _key_line_end(text: str, start: int) -> int: + """The end of the key line at `start` (without its \\r), or -1 if it isn't one. The last + line may end in the cut's "...".""" + line = _KEY_LINE.match(text, start) + if line is None or line.end() == start: + return -1 + end = line.end() + if end + 3 == len(text) and text.startswith("...", end): + return len(text) + carriage = text[end : end + 1] == "\r" + if end + carriage == len(text) or text[end + carriage : end + carriage + 1] == "\n": + return end + return -1 + + +def _named( + text: str, low: str, spans: list[tuple[int, int, str]], at_line: bool = False +) -> None: + if at_line: + # A value never runs past its line, so names at line starts never overlap. + for form in _FORMS: + if form in low: + for match in _NAMED_AT_LINE[form].finditer(low): + _take(text, match, spans) + return + found: list[tuple[int, str, re.Match[str]]] = [] + runs = 0 + for form in _FORMS: + if form not in low: + continue + before = len(found) + found.extend((m.start(), form, m) for m in _NAMED[form].finditer(low)) + if form in _NAMED_CAMEL: + found.extend( + (m.start(), form, m) + for m in _NAMED_CAMEL[form].finditer(low) + if text[m.start()].isupper() + ) + runs += len(found) > before + if runs > 1: + found.sort(key=_first) + at = 0 + for start, form, match in found: + # A name inside a matched value is not a start (r4 M7): overlapping values stay linear. + if ( + start >= at + and _name_sits(text, start, form) + and not _host_name(text, start, form) + ): + at = _take(text, match, spans) + + +def _host_name(text: str, start: int, form: str) -> bool: + """Whether the name at `start` ends a URL's host name followed by its port, as a Compose or + Kubernetes service name does (http://vault-secret:8200/v1/kv, cr1 M1): its ":" opens no + value. The port is checked first, so most names never look back for a host.""" + return ( + _HOST_PORT.match(text, start + len(form)) is not None + and _host_start(text, start) >= 0 + ) + + +def _take(text: str, match: re.Match[str], spans: list[tuple[int, int, str]]) -> int: + """Masks the value a name's match opens, unless it is a reference. Returns its end.""" + begin = match.end() + # The value's opening quote is the last group matched, if it has one. + quote = text[begin - 1] if match.lastgroup else "" + stop = (_QUOTE_END[quote] if quote else _PLAIN_END).search(text, begin) + finish = stop.start() if stop else len(text) + if _secret_value(text[begin:finish], bool(quote)): + spans.append((begin, finish, "named")) + return finish + + +def _first(found: tuple[int, str, re.Match[str]]) -> int: + return found[0] + + +def _name_sits(text: str, start: int, form: str) -> bool: + """Whether the name at `start` sits in its identifier as its kind allows.""" + kind, capital = _FORMS[form] + before = text[start - 1] if start else "" + joined = before == "_" or before == "-" + whole = not before or not (before.isalnum() or joined) + if kind == "upper": + return joined and text.startswith("PASS", start) and _upper_before(text, start) + if kind == "exact": + return whole or joined + camel = ( + text[start].isupper() + and before.isalnum() + and (not before.isupper() or text[start + 1 : start + 2].islower()) + ) + if kind == "compact": + return text[start + capital].isupper() and (whole or camel) + if form == "token" and _identifier(text, start, start + 5) in _PAGINATION: + return False + return whole or joined or camel + + +def _identifier(text: str, start: int, end: int) -> str: + # Bounded: no pagination name is longer than 21 characters. + begin = start + while ( + begin > 0 + and start - begin < 17 + and (text[begin - 1].isalnum() or text[begin - 1] == "_") + ): + begin -= 1 + return text[begin:end] + + +def _upper_before(text: str, start: int) -> bool: + # The identifier before PASS holds no lower-case letter, looked for at most 64 back. + begin = start + while ( + begin > 0 + and start - begin < 64 + and (text[begin - 1].isalnum() or text[begin - 1] == "_") + ): + begin -= 1 + if text[begin].islower(): + return False + return True + + +def _secret_value(value: str, quoted: bool) -> bool: + """Whether a value that passed the length test is masked: never when it is a reference.""" + if value.isalnum(): + # Letters and digits alone are neither a reference nor code. + return True + if ( + value.startswith(_REFERENCE_STARTS) + or "os.environ" in value + or "process.env" in value + or "env(" in value + ): + return False + return quoted or _CODE.match(value) is None + + +# ---------------------------------------------------------------------------------- chat +# A key is held whole up to this many characters, the cleaner's _HOLD_HARD; past it the +# cleaner's own long-URL rule applies. A key block has no such bound (r4 L7). +_LONGEST_HELD_KEY = 16_384 +_SPACE_RUN = re.compile(r"\s+") +_SEAM_NAME = re.compile(_NAME_ALTERNATION) +_SEAM_GUARD = re.compile("|".join(_GUARD_NAMES)) +_TAIL_PATTERN = re.compile(_TAIL) +# Every beginning of a tail: while what follows a name is one, its match is still open. +_TAIL_OPEN = re.compile( + r"[\"'`]?[*_`]{0,4}(?:[ \t]{0,16}|\s{0,4})(?:[=:][*_`]{0,4}(?:[ \t]{0,16}|\s{0,4}))?" +) +_GUARD_TAIL = re.compile(r"[\s:=]+['\"]?") +_GUARD_TAIL_OPEN = re.compile(r"[\s:=]*") +_GUARD_VALUE_END = re.compile(r"[^A-Za-z0-9/+=]") +_PLAIN_START = re.compile(_PLAIN_VALUE) +_USER_END = re.compile(r"[\s/?#@:\[\]]") +# Every beginning of rule 1's text after "user:", while no @ has come: a CRLF's \r alone too, +# when its \n is still to come in the next delta (cr2 M3). +_RULE1_OPEN = re.compile(rf"[^\s@]{{0,64}}(?:{_RUN}[^\s@]{{0,64}}|[ \t]{{0,4}}\r)?") +_HEADER_OPEN = re.compile(r"-----BEGIN[A-Z0-9 ]{0,60}-{0,5}") +_SEAM_KEY = _HOST_LITERAL +_URL_MARK = re.compile("://") +_LONGEST_END = len("-----END " + "X" * 40 + "PRIVATE KEY BLOCK-----") +# How far back a delta's search reaches into the text before it: the longest name, guard name +# or literal, less one, so one split across two deltas is still found. +_REACH = ( + max(_LONGEST_NAME, len("azure_storage_account_key"), *map(len, _KEY_LITERALS)) - 1 +) + +# A named value's phases in ChatHold. +_FRAME, _QUOTED, _PLAIN = range(3) + + +def _piece_start(piece: "_Piece") -> int: + return piece.start + + +class _Piece: + """A whitespace-separated piece of the stream.""" + + __slots__ = ("start", "end", "address", "after", "extends", "query_end") + + def __init__(self, start: int, before: "_Piece | None") -> None: + self.start = start + self.end = -1 + self.address = False + # The address piece this one follows, if any: a candidate here holds from there. + self.after = before if before is not None and before.address else None + # Whether rule 2 reaches into this piece from the address piece before it. + self.extends = False + # For an address piece: its query ends in & or ?, so rule 2 reaches the next piece. + self.query_end = False + + +class ChatHold: + """Where hosted chat may cut its streamed reply, so no rule here is split between two + pieces (design page, "The hold rule" and "The seam"). + + `_DeltaCleaner` holds the text after its last cut and cleans each piece it lets out. Rule 1, + rule 2, a named value and a key block match across whitespace, so the cleaner asks here how + far it may go: never past its own cut, never into a match that is still open or has just + closed, and never into an address piece holding a candidate, or the piece after one. + + The work per delta is bounded by the delta's length plus a constant (r4 H2, r1 M7): the + state is kept between deltas, and only the new text and a short tail are searched. A + candidate's match is decided once, when what closes it arrives. Positions count from the + start of the prose stream; the held text is always the end of it. + """ + + def __init__(self) -> None: + # Set by the cleaner, whose `_learn` alone decides it. Before prose the cleaner never + # cuts, since a lone URL broken by a line break is one URL (r4 M2), so neither does this. + self.prose = False + self._end = 0 + # Just after the last whitespace: the cleaner's own cut. + self._space_end = 0 + self._pieces: list[_Piece] = [] + # Open candidates, by where each starts: named values, guard names, rule 1's + # scheme://user: and key headers still arriving. + self._names: dict[int, list] = {} + self._guards: dict[int, list] = {} + self._users: dict[int, list] = {} + self._headers: dict[int, int] = {} + # Named values and guard names that have closed, so the tail search doesn't take them + # again. + self._done: set[int] = set() + # Matches that have closed, as (start, end, hold start): no cut falls inside one, so a + # match that has just closed goes out whole (r3 M3). + self._closed: list[tuple[int, int, int]] = [] + # An open key block: [hold start, header start, where END is next looked for]. + self._block: list[int] | None = None + # A block no END can close, since a later header came first: it holds to the end. + self._forever: int | None = None + # Where a key in the last, unfinished piece holds from. + self._key: int | None = None + # The last "://" and "-----BEGIN" taken, so each is taken once. + self._url_seen = -1 + self._header_seen = -1 + + def cut(self, held: str, delta: str) -> int: + """The index into `held + delta` up to which the text may be sent.""" + if not self.prose: + return 0 + base = self._end - len(held) + old_end = self._end + self._end += len(delta) + whole = held + delta + self._forget(base) + if self._forever is not None: + return 0 + self._split(whole, base, old_end) + start = max(base, old_end - _REACH) + while True: + if self._block is not None: + closed = self._block_end(whole, base) + if closed < 0: + break + start = max(start, closed) + if not self._look(whole, base, start): + break + if self._forever is not None: + return 0 + self._follow(whole, base) + return self._limit(base) - base + + def shed_cut(self, held: str) -> int | None: + """Where the cleaner's flush past its soft bound may cut `held`. + + None when nothing here holds the text, so the cleaner sheds as before; 0 when nothing may + be sent yet. A key block is held whatever its length; a key, a named value and a match + that has closed, up to the hard bound.""" + if not self.prose: + return None + base = self._end - len(held) + holds = list(self._headers.values()) + if self._forever is not None: + holds.append(self._forever) + if self._block is not None: + holds.append(self._block[0]) + if not holds and self._key is not None and len(held) <= _LONGEST_HELD_KEY: + holds.append(self._key) + if len(held) <= _LONGEST_HELD_KEY: + # A named value longer than the soft bound is held as a key is (cr1 M3): shed past + # it, the cleaned text before the cut would mask the value only that far, and the + # rest would go out later with no name in front of it. A match that has closed is + # held too: one still held waits on the piece after its URL, and the branches would + # cut it at that URL's start, between its name and its value. + holds += [state[4] for state in self._names.values()] + holds += [state[4] for state in self._guards.values()] + holds += [hold for _, end, hold in self._closed if end > base] + if not holds: + return None + return max(0, min(holds) - base) + + # -- the stream --------------------------------------------------------------------------- + + def _forget(self, base: int) -> None: + # What the cleaner has sent or dropped since the last delta is no longer held. + self._closed = [span for span in self._closed if span[1] > base] + for found in (self._names, self._guards, self._users, self._headers): + for start in [start for start in found if start < base]: + del found[start] + self._done = {start for start in self._done if start >= base} + if self._block is not None and self._block[1] < base: + self._block = None + pieces = self._pieces + gone = 0 + while gone < len(pieces) - 1 and -1 < pieces[gone].end <= base: + gone += 1 + if gone: + del pieces[:gone] + if self._key is not None and self._key < base: + self._key = base + + def _split(self, whole: str, base: int, old_end: int) -> None: + """Brings the pieces up to date with the delta.""" + at = old_end + for space in _SPACE_RUN.finditer(whole, old_end - base): + begin = space.start() + base + if begin > at: + self._grow(whole, base, at, begin) + self._close(whole, base, begin) + self._space_end = at = space.end() + base + if at < self._end: + self._grow(whole, base, at, self._end) + + def _grow(self, whole: str, base: int, start: int, end: int) -> None: + pieces = self._pieces + if not pieces or pieces[-1].end != -1: + before = pieces[-1] if pieces else None + piece = _Piece(start, before) + if piece.after is not None: + piece.extends = before.query_end or whole[start - base] in "?&#" + pieces.append(piece) + piece = pieces[-1] + if not piece.address: + piece.address = ( + "://" in whole[max(piece.start, start - 2, base) - base : end - base] + ) + + def _address(self, whole: str, base: int, end: int) -> None: + """Marks the piece a rule 1 match ends in as an address piece (cr2 M2). A match that + crossed whitespace ends in a piece with no :// of its own (`ss@h:6379` in + `redis://:pa ss@h:6379`), which is the URL the whole text's rule 1 leaves, so rule 2 may + reach the piece after it and that piece holds from it.""" + pieces = self._pieces + # The last piece starting before `end`, found by bisection: a walk back from the end + # made one large delta with many such matches quadratic (cr3 M1). + index = bisect_left(pieces, end, key=_piece_start) - 1 + if index < 0: + return + piece = pieces[index] + if piece.address: + return + piece.address = True + if piece.end == -1: + return + # It closed in this delta, so what `_close` and `_grow` decided from it is decided again. + text = whole[max(piece.start, base) - base : piece.end - base] + piece.query_end = text[-1:] in ("&", "?") and "?" in text + if index + 1 < len(pieces) and base <= pieces[index + 1].start < self._end: + following = pieces[index + 1] + following.after = piece + following.extends = ( + piece.query_end or whole[following.start - base] in "?&#" + ) + if following.extends and following.end != -1: + self._closed.append((piece.start, following.end, piece.start)) + + def _close(self, whole: str, base: int, end: int) -> None: + pieces = self._pieces + if not pieces or pieces[-1].end != -1: + return + piece = pieces[-1] + piece.end = end + # A key's characters hold no whitespace: one in this piece has closed. + self._key = None + if piece.address: + text = whole[max(piece.start, base) - base : end - base] + piece.query_end = text[-1:] in ("&", "?") and "?" in text + if piece.extends: + # Rule 2 has matched: the address piece and this one go out together. + self._closed.append((piece.after.start, end, piece.after.start)) + + def _hold_of(self, at: int) -> int: + """Where a candidate at `at` holds from: its piece's start, or the start of the address + piece it is in or follows, so a URL's path and query stay whole (r2 H1, r4 M1).""" + for piece in reversed(self._pieces): + if piece.start <= at: + if piece.address: + return piece.start + if piece.after is not None: + return piece.after.start + return piece.start + return at + + # -- candidates --------------------------------------------------------------------------- + + def _look(self, whole: str, base: int, start: int) -> bool: + """Takes the candidates that start at `start` or later. True when a key header opened a + block: the text after it is then left to the block.""" + begin = start - base + stop = len(whole) + opened = False + found = whole.find("-----BEGIN", max(begin, self._header_seen + 1 - base)) + while found >= 0: + self._header_seen = found + base + header = _PEM.match(whole, found) + if header is not None: + self._block = [ + self._hold_of(found + base), + found + base, + header.end() + base, + ] + stop, opened = found, True + break + if _HEADER_OPEN.fullmatch(whole, found): + self._headers[found + base] = self._hold_of(found + base) + found = whole.find("-----BEGIN", found + 1) + self._look_urls(whole, base, begin, stop) + self._look_keys(whole, base, begin, stop) + self._look_names(whole, base, begin, stop) + return opened + + def _look_urls(self, whole: str, base: int, begin: int, stop: int) -> None: + for mark in _URL_MARK.finditer( + whole, max(begin, self._url_seen + 1 - base), stop + ): + at = mark.start() + self._url_seen = at + base + scheme = _SCHEME_RUN_BEFORE.search(whole, max(0, at - 64), at) + first = scheme.start() if scheme else at + if first < at and whole[first].isalpha(): + # Rule 1 may start here; its user runs to the first of _USER_END. + self._users[first + base] = [ + first + base, + at + 3 + base, + -1, + self._hold_of(at + base), + ] + + def _look_keys(self, whole: str, base: int, begin: int, stop: int) -> None: + pieces = self._pieces + if not pieces or pieces[-1].end != -1: + return + for key in _SEAM_KEY.finditer(whole, max(begin, pieces[-1].start - base), stop): + hold = self._hold_of(key.start() + base) + self._key = hold if self._key is None else min(self._key, hold) + + def _look_names(self, whole: str, base: int, begin: int, stop: int) -> None: + low = whole[begin:stop].translate(_ASCII_LOWER) + for pattern, found in ((_SEAM_NAME, self._names), (_SEAM_GUARD, self._guards)): + for name in pattern.finditer(low): + at = begin + name.start() + base + if at not in found and at not in self._done: + found[at] = [ + _FRAME, + at + len(name.group()), + -1, + "", + self._hold_of(at), + ] + + def _block_end(self, whole: str, base: int) -> int: + """Looks for the open block's END in the new text and a tail as long as END's longest + form. Returns where the block closed, or -1 while it is open.""" + block = self._block + start = max(block[2] - _LONGEST_END, block[1] + 1) - base + end = _PEM_END.search(whole, start) + limit = end.start() if end is not None else len(whole) + later = whole.find( + "-----BEGIN", max(start, self._header_seen + 1 - base), limit + ) + while later >= 0: + self._header_seen = later + base + if _PEM.match(whole, later) or _HEADER_OPEN.fullmatch(whole, later): + # Another header before any END: an END pairs with the nearest header before + # it (r4 L4), so none can close this block. It holds to the message's end. + self._forever = block[0] + return -1 + later = whole.find("-----BEGIN", later + 1, limit) + if end is None: + block[2] = self._end + return -1 + self._closed.append((block[1], end.end() + base, block[0])) + self._block = None + return end.end() + base + + def _follow(self, whole: str, base: int) -> None: + """Moves each open candidate on by what the delta brought, searching new text only.""" + for found, step in ( + (self._names, self._name_step), + (self._guards, self._guard_step), + ): + for at, state in list(found.items()): + closed = step(whole, base, at, state) + if closed is None: + continue + del found[at] + if closed > 0: + self._done.add(at) + self._closed.append((at, closed, state[4])) + for at, state in list(self._users.items()): + closed = self._user_step(whole, base, state) + if closed is None: + continue + del self._users[at] + if closed > 0: + self._closed.append((at, closed, state[3])) + self._address(whole, base, closed) + for at in list(self._headers): + if _PEM.match(whole, at - base) is not None: + hold = self._headers.pop(at) + if self._block is None: + self._block = [hold, at, at + 1] + else: + self._forever = self._block[0] + elif not _HEADER_OPEN.fullmatch(whole, at - base): + del self._headers[at] + + def _name_step(self, whole: str, base: int, at: int, state: list) -> int | None: + """None while the named value may still match; its end once it closes; 0 when it can't + match.""" + if state[0] == _FRAME: + low = whole[at - base : state[1] - base + _LONGEST_NAME].translate( + _ASCII_LOWER + ) + name = _SEAM_NAME.match(low) + if name is None: + return 0 + name_end = at - base + len(name.group()) + if _TAIL_OPEN.fullmatch(whole, name_end): + return None + if whole.startswith("::", name_end): + return 0 + tail = _TAIL_PATTERN.match(whole, name_end) + if tail is None: + return 0 + value = tail.end() + if value >= len(whole): + return None + if whole[value] in "\"'`": + state[0], state[2], state[3] = _QUOTED, value + 1 + base, whole[value] + elif _PLAIN_START.match(whole, value): + state[0], state[2] = _PLAIN, value + base + else: + return 0 + if state[0] == _QUOTED: + stop = _QUOTE_END[state[3]].search(whole, state[2] - base) + if stop is None: + state[2] = self._end + return None + closed = stop.end() if whole[stop.start()] == state[3] else stop.start() + return closed + base + stop = _PLAIN_END.search(whole, state[2] - base) + if stop is None: + state[2] = self._end + return None + return stop.start() + base + + def _guard_step(self, whole: str, base: int, at: int, state: list) -> int | None: + """As `_name_step`, for aws_secret_key and azure_storage_key's own names and tail.""" + if state[0] == _FRAME: + name_end = state[1] - base + if _GUARD_TAIL_OPEN.fullmatch(whole, name_end): + return None + tail = _GUARD_TAIL.match(whole, name_end) + if tail is None: + return 0 + if tail.end() >= len(whole): + return None + state[0], state[2] = _PLAIN, tail.end() + base + stop = _GUARD_VALUE_END.search(whole, state[2] - base) + if stop is None: + state[2] = self._end + return None + return stop.start() + base + + def _user_step(self, whole: str, base: int, state: list) -> int | None: + """Rule 1 from this scheme: None while it may still match, its end once it does, 0 when + it can't.""" + if state[2] < 0: + stop = _USER_END.search(whole, state[1] - base) + if stop is None: + state[1] = self._end + return None + if whole[stop.start()] != ":": + return 0 + state[1] = state[2] = stop.end() + base + at = whole.find("@", state[1] - base) + if at < 0: + state[1] = self._end + return None if _RULE1_OPEN.fullmatch(whole, state[2] - base) else 0 + match = _RULE1.match(whole, state[0] - base) + return match.end() + base if match is not None else 0 + + def _limit(self, base: int) -> int: + """The cleaner's own cut, moved back before each open candidate's hold start and out of + each closed match.""" + holds = [state[4] for state in self._names.values()] + holds += [state[4] for state in self._guards.values()] + holds += [state[3] for state in self._users.values() if state[2] >= 0] + holds += self._headers.values() + if self._block is not None: + holds.append(self._block[0]) + if self._key is not None: + holds.append(self._key) + if self._pieces: + last = self._pieces[-1] + if last.end != -1 and last.address: + # Rule 2 may yet reach the piece after it. + holds.append(last.start) + elif last.end == -1 and last.extends: + holds.append(last.after.start) + cut = min(self._space_end, *holds) if holds else self._space_end + moved = True + while moved: + moved = False + for start, end, hold in self._closed: + if start < cut < end and hold < cut: + cut, moved = hold, True + return max(cut, base) diff --git a/tests/harness/test_builder_chat.py b/tests/harness/test_builder_chat.py index 703539a07..12a1f7ff2 100644 --- a/tests/harness/test_builder_chat.py +++ b/tests/harness/test_builder_chat.py @@ -2707,3 +2707,1365 @@ def test_random_splits_never_send_a_credential_the_stripped_whole_drops(): if leaked: found.append((text, leaked)) assert found == [] + + +# -- TH-9500: the hold rule for secrets matched across whitespace (plan v1.1, C1-C4) -------------- +# Every key-shaped value is fake. The cases are the design reviewers' lists, copied as data. + +_GH = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8" +_SKP = "sk-proj-" + "FAKEfake0123456789abcdefFAKEfake0123456789ab" +_AK = "AKIA" + "FAKEFAKE12345678" +_XB = "xoxb-12345678901-12345678901-" + "FAKEfakeFAKEfakeFAKEfake" +_SLACK = "https://hooks.slack.com/services/" + "T00FAKE00/B00FAKE00/" + "FAKEfakeFAKEfakeFAKEfake" +_TG = "https://api.telegram.org/bot123456789:AAFAKEfakeFAKEfakeFAKEfakeFAKEfake_x/sendMessage" +_JWT = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJmYWtlIn0.FAKEsigFAKEsig" +_PEM = "-----BEGIN RSA PRIVATE KEY-----\nMIIEFAKEFAKEFAKE\nABCDEFFAKE==\n-----END RSA PRIVATE KEY-----" +# Round 1's cases (r1cases.py) and the addendum's three: round 2's 1,919 stream splits. +_ROUND1 = { + "page: ws in password": "https://bot:hun ter2@h/v1", + "page: lb in password": "see https://bot:hun\n ter2@h/v1 is the hook", + "page: tab in password": "see https://bot:hun\tter2@h/v1 is the hook", + "page: query next line": "see https://u:pw@h/x\n ?key=sk-1 and more", + "page: ?# in password": "https://bot:8812#x@db/", + "page: / in password": "https://bot:pa/ss@db", + "page: :// in password": "https://bot:pa://ss@db", + "bare ghp prose": f"Use {_GH} to push.", + "bare sk-proj prose": f"key is {_SKP}.", + "bare AKIA quoted": f'aws="{_AK}"', + "bare xoxb json": f'{{"t": "{_XB}"}}', + "slack hook": f"Post to {_SLACK} daily", + "telegram": f"Call {_TG} now", + "jwt prose": f"Bearer {_JWT}", + "pem": f"key:\n{_PEM}\nend", + "pem header only": "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAAFAKE\n(truncated)", + "assign pw": "password=hunter22", + "assign spaced": "token: s3cr3tvalue99", + "tab query": "see https://h/x\t?key=sk-1 more", + "mid-query break": "see https://h/x?a=1&\n key=sk-1 more", + "break before ? no indent": "see https://h/x\n?key=sk-1 more", + "nbsp query": "see https://h/x\xa0?key=sk-1 more", + "nbsp password": "https://bot:hun\xa0ter2@h/v1", + "H port+email": "Run at http://localhost:3000 and email support@acme.com", + "H query email": "https://host:8080/path?email=a@b.com", + "H npm scoped": "Install from https://registry.npmjs.org:443/@babel/core today", + "H medium": "Read https://medium.com/@user for more", + "H port then mention": "Open http://localhost:8080 and ping @oncall in Slack", + "H md amp": "See https://docs.x.com/a\n©=2024 Acme", + "H password required": "password: required", + "H token expires": "the token: expires hourly", + "H secret never": "secret: never tell anyone", + "H env": 'api_key=os.environ["OPENAI_API_KEY"]', + "H env placeholder": "Send the token: ${SLACK_BOT_TOKEN} in the header", + "H commit": "Fixed in 1ac329e16b0f2c9a8e7d6c5b4a3f2e1d0c9b8a7f", + "H uuid": "order 123e4567-e89b-12d3-a456-426614174000", + "H b64img": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", + "H order": "Order ID ORD-2024-000123 and SKU AKIAMODEL", + "H token count": "max_tokens: 4096 and token=streaming", + "a1": 'config "password": "Zq8wYv3Lp9" ok', + "a3": "use https://bot:hun ter2@h/v1 now", + "a4": f"key:\n{_PEM}\nend", +} +# The design page's chat-test list, each with trailing text. +_RUN = "/".join(["a" * 50] * 90) +_CHAT_CASES = { + "ghp in a long run": '{"x":"' + + _RUN[:3000] + + '","t":"' + + _GH + + '","y":"' + + _RUN[:2000] + + '"} done now', + "named value at its ': '": 'config "password": "Zq8wYv3Lp9" ok then more', + "rule 1 at its space": "use https://bot:hun ter2@h/v1 now and later", + "rule 1 at a break + 4": "see https://bot:hun\n ter2@h/v1 is the hook", + "block": f"key:\n{_PEM}\nend of message here", + "block in YAML": f"tls:\n key: |\n {_PEM.replace(chr(10), chr(10) + ' ')}\n ok: true", + "block CRLF": f"Here:\r\n{_PEM.replace(chr(10), chr(13) + chr(10))}\r\nKeep it safe.", + "block encrypted": "Here:\n-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: " + "AES-128-CBC,3F17F5316E2BAC89\n\nMIIEFAKEQUJDREVG\n-----END RSA PRIVATE KEY-----\nKeep it.", + "block one line": "Use this: -----BEGIN RSA PRIVATE KEY----- MIIEFAKEQUJDREVG " + "-----END RSA PRIVATE KEY----- then restart", + "login query": "Login at https://h.example/login?user=bob&password=Sunshine then stop", + "oauth code": "POST https://oauth2.googleapis.com/token?code=4/0AXfakeFAKEcode123&grant_type=x and wait", + "telegram path": f"Call {_TG}?chat_id=1 now please", + "a URL holding a name and a prefix": "Open https://h.example/cb/token?ref=sk-not-a-key&session=Qm9vZmFrZQ then go", + "rule 2 on the next line": "see https://h/x?a=1\n&token=Sunshine and more", + "named md label": "- **Password:** Zq8wYv3Lp9 then go on", + "named camel json": '{"clientSecret": "Zq8wYv3Lp9Kd2xT", "x": 1} done now', + "named env pass": "DB_PASS=Zq8wYv3Lp9\nSMTP_PASS=hunter22x9\nok", + "named 16 spaces": 'admin = "x"\npassword = "Zq8wYv3Lp9" ok', + "named kebab": "cloud.aws.credentials.secret-key: Zq8wYv3Lp9Kd2xT ok", + "named quoted with spaces": 'cfg "password": "correct horse battery 9" ok more', + "named break + 4": "password:\n Zq8wYv3Lp9 ok", + "conn string": "Use DefaultEndpointsProtocol=https;AccountName=acct;AccountKey=FAKEazureFAKEkey0123" + "456789abcdef==;EndpointSuffix=core.windows.net then go", + "azure key, spaced": "set AZURE_KEY = " + + ("FAKEazureFAKEkey0123456789abcdefFAKE+/" * 3)[:86] + + "== ok", +} +_PLANTED = [ + "hun ter2", + "ter2@", + "hunter22", + "Zq8wYv3Lp9", + "s3cr3tvalue99", + "MIIEFAKE", + "ABCDEFFAKE", + _GH, + _SKP, + "battery 9", + "FAKEazureFAKEkey0123", + "Sunshine", + "4/0AXfake", + "Qm9vZmFrZQ", + "AAFAKEfake", + "key=sk-1", + "FAKEfakeFAKEfakeFAKEfake", + "FAKEsigFAKEsig", + "QUJDREVG", +] + + +def _events_of(deltas: list[str]) -> list[str]: + from fi.alk.harness.hosted_chat_entrypoint import _DeltaCleaner + + cleaner = _DeltaCleaner() + events = [cleaner.feed(delta) for delta in deltas] + events.append(cleaner.flush()) + return [event for event in events if event] + + +def _removed_fragments(text: str, whole: str) -> set[str]: + """k1_stream's frags(): 6-character fragments of what the whole clean removes, less any + that also occur in what it keeps (r4 L8).""" + import difflib + + matcher = difflib.SequenceMatcher(None, text, whole, autojunk=False) + removed = [ + text[start:end] + for op, start, end, _, _ in matcher.get_opcodes() + if op in ("delete", "replace") and len(text[start:end].strip()) >= 6 + ] + fragments = { + piece[at : at + 6] for piece in removed for at in range(len(piece) - 5) + } + return {fragment for fragment in fragments if fragment not in whole} + + +def _split_problems(text: str, splits: list[list[str]]) -> list[str]: + """C1's three checks, for each way of splitting `text` into deltas: the joined events equal + one whole clean; no 6-character fragment of a removed string is in any event; no planted + secret the whole clean drops is in the joined events.""" + from fi.alk.providers.agent import _redact + + whole = _redact(text) + fragments = _removed_fragments(text, whole) + problems = [] + for deltas in splits: + events = _events_of(deltas) + joined = "".join(events) + if joined != whole: + problems.append(f"unequal at {[len(d) for d in deltas][:3]}") + elif any( + event[at : at + 6] in fragments + for event in events + for at in range(len(event) - 5) + ): + problems.append(f"a fragment at {[len(d) for d in deltas][:3]}") + elif any( + secret in text and secret not in whole and secret in joined + for secret in _PLANTED + ): + problems.append(f"a planted secret at {[len(d) for d in deltas][:3]}") + return problems + + +def _every_split(text: str) -> list[list[str]]: + return [[text[:at], text[at:]] for at in range(1, len(text))] + [list(text)] + + +def test_round_twos_1919_stream_splits_each_equal_one_whole_clean() -> None: + """C2 (W3): round 1's cases and the addendum's, every two-way split and char by char.""" + splits = {name: _every_split(text) for name, text in _ROUND1.items()} + assert sum(len(each) for each in splits.values()) == 1919 + + problems = { + name: _split_problems(_ROUND1[name], each) for name, each in splits.items() + } + + assert {name: found[:3] for name, found in problems.items() if found} == {} + + +@pytest.mark.parametrize("name", list(_CHAT_CASES)) +def test_each_chat_case_split_at_every_offset_equals_one_whole_clean(name: str) -> None: + """C1: every offset (sampled in the long run), and 1-, 3- and 7-character deltas.""" + text = _CHAT_CASES[name] + offsets = range(1, len(text)) if len(text) < 2000 else range(1, len(text), 7) + splits = [[text[:at], text[at:]] for at in offsets] + splits += [ + [text[at : at + size] for at in range(0, len(text), size)] for size in (1, 3, 7) + ] + + assert _split_problems(text, splits)[:3] == [] + + +def test_every_event_the_pump_sends_masks_a_secret_split_across_its_whitespace( + tmp_path, monkeypatch +): + """C1 through _pump_events: assistant_delta, tool_result and the turn's assistant_message + (the question is below). A named value split at its ': ' and a password at its space.""" + from fi.alk.providers.agent import _redact + + client = BuilderClient() + runtime = _runtime(tmp_path, client, monkeypatch) + queue: asyncio.Queue = asyncio.Queue() + said = 'Set "password": "Zq8wYv3Lp9" and use https://bot:hun ter2@h.example/v1 then go.' + colon, space = said.index(": ") + 1, said.index("hun ") + 4 + pieces = [ + said[:colon], + said[colon : colon + 1], + said[colon + 1 : space], + said[space:], + ] + for piece in pieces: + queue.put_nowait( + RuntimeEvent(kind="text", text=piece, detail={"partial": True}) + ) + queue.put_nowait( + RuntimeEvent(kind="tool", tool="mcp__x__Read", detail={"call_id": "r"}) + ) + queue.put_nowait( + RuntimeEvent( + kind="result", text=f".env:1: GITHUB_TOKEN={_GH}", detail={"call_id": "r"} + ) + ) + queue.put_nowait(None) + + asyncio.run( + runtime._pump_events( + queue, + assistant_message_id="a-1", + fallback_invocation_id="turn-1", + emitted_text=[], + turn_failures=[], + ) + ) + + shown = json.dumps(client.events) + assert "Zq8wYv3Lp9" not in shown and "ter2" not in shown and _GH[4:12] not in shown + assert "".join(_deltas(client)) == _redact(said) + [result] = [one["payload"] for one in client.events if one["kind"] == "tool_result"] + assert result["text"] == ".env:1: GITHUB_TOKEN=[redacted]" + + +def test_the_turns_assistant_message_masks_a_key_split_between_its_pieces( + tmp_path, monkeypatch +): + client = BuilderClient() + runtime = _runtime(tmp_path, client, monkeypatch) + runtime._published_fingerprint() + runtime.conversation = _Saying( + tmp_path, _texts(["Saved password: ", "Zq8wYv3Lp9 ", "for you."]) + ) + + _turn(runtime, {"content": "save it"}) + + assert "Zq8wYv3Lp9" not in json.dumps(client.events) + assert "".join(_deltas(client)) == "Saved password: [redacted] for you." + assert _reply(client) == "Saved password: [redacted] for you." + + +def test_a_question_the_model_asks_masks_a_named_value(tmp_path, monkeypatch): + client = BuilderClient() + runtime = _runtime(tmp_path, client, monkeypatch) + runtime.stopping.set() + + asyncio.run( + runtime._ask( + "AskUserQuestion", + { + "question": "Keep DB_PASSWORD=Zq8wYv3Lp9 ?", + "options": [f"use {_GH}", "drop it"], + }, + None, + ) + ) + + asked = next(one for one in client.events if one["kind"] == "question_requested") + assert asked["payload"] == { + "prompt": "Keep DB_PASSWORD=[redacted] ?", + "options": ["use [redacted]", "drop it"], + } + + +def _seconds(deltas: list[str]) -> float: + import time + + best = 9.0 + for _ in range(3): + start = time.perf_counter() + _events_of(deltas) + best = min(best, time.perf_counter() - start) + return best + + +def test_the_seams_work_per_delta_is_bounded_by_the_delta() -> None: + """C3 (r4 H2, r1 M7): each under 0.5 s, best of 3.""" + words = "The quick brown fox jumps over the lazy dog while the build runs. " + refuse = ( + "If a user pastes -----BEGIN PRIVATE KEY----- text, refuse. " + + (words * 700)[: 40 * 1024] + ) + # 10,000 pairs, so a build that looks over the whole held text on every delta shows. + no_space = "Here: " + "a/" * 10_000 + _GH + " done" + saml = ( + "Sign in at https://idp.example.com/sso?SAMLRequest=" + + ("fZJNT8MwDIbvSPyHKPeuWwdbDEFAKE" * 160)[:4200] + + "&tok=" + + _GH + + "&RelayState=Sunshine then come back" + ) + quoted = 'cfg "password": "' + "a b c " * 500 + '" ok' + named_lines = ("token: Zq8wYv3Lp9 and password = abc\n" * 1200)[: 40 * 1024] + + times = { + "honest BEGIN + 40 KiB, 4-char deltas": _seconds( + [refuse[i : i + 4] for i in range(0, len(refuse), 4)] + ), + "20,000 with no space + ghp_, 1-char deltas": _seconds(list(no_space)), + "4 KB SAML query past N, 1-char deltas": _seconds(list(saml)), + "3 KB quoted value, 4-char deltas": _seconds( + [quoted[i : i + 4] for i in range(0, len(quoted), 4)] + ), + "a name on every line, 40 KiB, 4-char deltas": _seconds( + [named_lines[i : i + 4] for i in range(0, len(named_lines), 4)] + ), + } + + assert {name: seconds for name, seconds in times.items() if seconds >= 0.5} == {} + + +def test_a_lone_webhook_broken_by_a_line_break_before_prose_is_cleaned_as_one_url() -> ( + None +): + """C4 (r4 M2): before the text is known to be prose, the seam never cuts at \\n.""" + from fi.alk.providers.agent import _redact + + text = "https://hooks.slack.com/services/" + "T00FAKE00/B00FAKE00/\nFAKEfakeFAKEfakeFAKEfake" + for size in (1, 3, 7): + joined = "".join( + _events_of([text[i : i + size] for i in range(0, len(text), size)]) + ) + assert joined == _redact(text) == "https://hooks.slack.com/services/[redacted]" + + +@pytest.mark.parametrize("at", [4058, 4070, 4080, 4090, 4095]) +def test_a_key_straddling_the_soft_bound_in_a_held_url_query_never_streams( + at: int, +) -> None: + """C4 (r4 M1, r1 L6), at 1-character deltas: no head of the key in any event, and the honest + word after the URL ("then") is kept.""" + from fi.alk.providers.agent import _redact + + url = "https://idp.example.com/sso?SAMLRequest=" + filler = ("fZJNT8MwDIbvSPyHKPeuWwdbDEFAKE" * 160)[: at - len(url) - len("&tok=")] + text = ( + "Sign in at " + + url + + filler + + "&tok=" + + _GH + + "&RelayState=Sunshine then come back" + ) + + events = _events_of(list(text)) + + assert "".join(events) == _redact(text) + assert "".join(events).endswith(" then come back") + assert not any(_GH[:6] in event for event in events) + + +def test_a_key_block_longer_than_the_hard_bound_is_held_never_streamed() -> None: + """C4 (r4 L7): an 18 KB block is held whole, whatever M is.""" + from fi.alk.providers.agent import _redact + + lines = [f"L{i:03d}FAKEQUJDREVGR0hJSktM" + "A" * 36 for i in range(300)] + text = ( + "Key:\n-----BEGIN RSA PRIVATE KEY-----\n" + + "\n".join(lines) + + "\n-----END RSA PRIVATE KEY-----\nKeep it safe." + ) + assert len(text) > 16_384 + + events = _events_of([text[i : i + 7] for i in range(0, len(text), 7)]) + + assert "".join(events) == _redact(text) == "Key:\n[redacted]\nKeep it safe." + assert not any(line[4:30] in event for line in lines for event in events) + + +# -- TH-9500 change review r1 (cr1): what streams past the soft bound --------------------------- + +# cr1 M3: a named value longer than N, plain, quoted with spaces, AWS's guard name, and a URL as +# the value. Past N the shed sent the value cleaned only as far as it was held, then the rest +# with no name in front of it, or the URL whole. +_LONG_NAMED = { + "plain": "Note token: " + "Zq8w" * 1100 + " ok then", + "quoted with spaces": 'cfg "password": "' + "a1 b2 c3 " * 540 + '" ok then', + "aws guard": "aws_secret_access_key = " + + "wJalrXUtnFEMIFAKEK7MD" * 210 + + " ok then", + "a URL as the value": "Note token: https://h.example/" + "a" * 4200 + " ok then", +} + + +@pytest.mark.parametrize("name", list(_LONG_NAMED)) +def test_a_named_value_longer_than_the_soft_bound_streams_as_one_whole_clean( + name: str, +) -> None: + """cr1 M3: held as a key is, up to the hard bound, at 1-, 5- and 50-character deltas.""" + text = _LONG_NAMED[name] + splits = [ + [text[at : at + size] for at in range(0, len(text), size)] + for size in (1, 5, 50) + ] + + assert _split_problems(text, splits) == [] + + +@pytest.mark.parametrize("at", [4070, 4090]) +@pytest.mark.parametrize("size", [1, 5]) +def test_a_key_straddling_the_soft_bound_in_a_run_with_no_space_never_streams( + at: int, size: int +) -> None: + """cr1 M5 (round 3's w3_flush shape): a ghp_ key in a whitespace-free run, across the + 4,096th held character. The key is held whole, up to the hard bound.""" + from fi.alk.providers.agent import _redact + + text = "Here: " + ("a/" * 3000)[: at - 6] + _GH + "/" + "b/" * 300 + " done now" + assert text.find(_GH) == at + + events = _events_of([text[i : i + size] for i in range(0, len(text), size)]) + + assert "".join(events) == _redact(text) + assert not any(_GH[:12] in event or _GH[-20:] in event for event in events) + + +def test_the_seam_stays_linear_on_an_honest_begin_then_256_kib_of_prose() -> None: + """cr1 M6: the per-delta END search is bounded by the delta (r4 H2), at 256 KiB, best of + 3, under 1.5 s. Searched from the block's header on every delta, it took 3.5 s.""" + words = "The quick brown fox jumps over the lazy dog while the build runs. " + refuse = ( + "If a user pastes -----BEGIN PRIVATE KEY----- text, refuse. " + + (words * 4100)[: 256 * 1024] + ) + + seconds = _seconds([refuse[i : i + 4] for i in range(0, len(refuse), 4)]) + + assert seconds < 1.5, f"{seconds:.3f} s" + + +def test_text_after_an_end_split_across_two_deltas_streams_before_the_flush() -> None: + """cr1 L3: an END line split between two deltas closes the block when its second half + arrives, so what follows streams then, not only at the message's end.""" + from fi.alk.harness.hosted_chat_entrypoint import _DeltaCleaner + + text = f"key:\n{_PEM}\nThen restart the service and check the logs." + split = text.index("-----END") + len("-----END RSA PRI") + after = text.index("Then") + cleaner = _DeltaCleaner() + + streamed = "".join( + cleaner.feed(delta) for delta in (text[:split], text[split:after], text[after:]) + ) + + assert streamed.startswith("key:\n[redacted]\nThen restart the service and") + assert ( + streamed + cleaner.flush() + == "key:\n[redacted]\nThen restart the service and check the logs." + ) + + +# -- TH-9500 rule 1b: an @ before the password's / (design addendum v2.5.2) ---------------------- +# The rows test_secret_shapes.py's R1B_CLEANED and R1B_KEPT check on the whole text, copied as +# data. 1b's span and its host lookahead cross no whitespace and no ://, so ChatHold is +# unchanged (L5); a one-URL value runs step 1 on its joined token, as `_clean` does (M3, M-A). +_R1B_ROWS = [ + "postgres://admin:Xk9/Qp+Zr8@db.example.com:5432/app", + "https://user:pa#ss@host.example/x", + "postgres://admin:P@ss/Xk9@db.example.com:5432/app", + "smtp://me@gmail.com:app/pass@smtp.gmail.com:587", + "redis://:p@w/x9@cache.internal:6379/0", + "postgres://admin:P@ss:99/Xk9@db.example.com/app", + "postgres://admin:P@ss/Xk9@localhost/app", + "postgres://admin:P@ss/Xk9@db/app", + "postgres://admin:P@ss/Xk9@[::1]:5432/app", + "postgres://admin:P@ss/Xk9@postgres_db:5432/app", + "postgres://admin:P@ss/Xk9@my_db/app", + "`redis://:p@w/x9@redis`", + "postgres://admin:P@ss/Xk9@db:${DB_PORT}/app", + "postgres://admin:P@ss/Xk9@db:$DB_PORT/app", + "postgres://admin:P@ss/Xk9@db?sslmode=disable", + "redis://:p@w/x9@redis#0", + "(amqp://u:p@w/x9@rabbitmq)", + "amqp://u:p@w/x9@rabbitmq, then", + "amqp://u:p@w/x9@rabbitmq; then", + "", + "[amqp://u:p@w/x9@rabbitmq]", + "postgres://admin:P@ss/Xk9@[fe80::1%25en0]:5432/app", + "a://u:x://v:p@h/x@host.example", + "http://admin@localhost:8080/bob@example.com", + "ssh://git@gitlab.example.com:2222/ops@infra.example/repo.git", + "https://ci@registry.example.com:8443/lodash@4.17.21", + "ssh://git@host.example:22/org@team/repo", + "http://admin@[::1]/bob@example.com", + "ssh://git@[fd00::5]/org@team/repo", + "https://ci@registry.example.com/lodash@4.17.21", + "ssh://git@github.com/org@team/repo", + "Mail https://u:p@h.example/to@~team now", +] +# M3 and M-A: equal and clean when the URL stands alone (in prose, a break inside the span is a +# named limit). +_R1B_BROKEN = { + "postgres://admin:P@ss/Xk\n9@db.example.com:5432/app": "postgres://db.example.com:5432/app", + "postgres://admin:Pa\nss@w/x9@db.example.com/app": "postgres://db.example.com/app", + "redis://:pa\nss@w/x9@cache.internal:6379/0": "redis://cache.internal:6379/0", + "a://u:x://v:p@h/\nx@host.example": "a://host.example", +} + + +@pytest.mark.parametrize("text", _R1B_ROWS, ids=range(len(_R1B_ROWS))) +def test_rule_1b_streams_as_the_whole_text_cleans_at_every_split(text: str) -> None: + """Alone and in prose, every two-way split and char by char, with C1's three checks.""" + for each in (text, f"Use {text} now, then {text} again."): + assert _split_problems(each, _every_split(each))[:3] == [], each + + +@pytest.mark.parametrize("text", list(_R1B_BROKEN), ids=range(len(_R1B_BROKEN))) +def test_a_one_url_value_broken_inside_1bs_span_streams_equal_and_clean( + text: str, +) -> None: + from fi.alk.providers.agent import _redact + + assert _redact(text) == _R1B_BROKEN[text] + assert _split_problems(text, _every_split(text)) == [] + + +# -- TH-9500 change review r2 (cr2) ----------------------------------------------------------------- + +# cr2 H1: a reply that is one URL, with a key or a named value at the start of a later line. +# Cleaning the whole text masks it before the join; before prose, so does the stream. Each row +# is (reply, the secret that must not stream). Owl's ten shapes, then rows longer than N, which +# go through `_shed`'s branch before prose. +_AKIA = "AKIA" + "FAKEFAKEFAKE1234" +_JWT_LONG = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJmYWtlIn0.FAKEsigFAKEsigFAKEsig" +_LONG_URL = "https://h.example/" + "a" * 4100 +_LINE_STARTS = { + "url \\n ghp": ("https://github.com/settings/tokens\n" + _GH, _GH), + "url \\t ghp": ("https://github.com/settings/tokens\t" + _GH, _GH), + "url \\r\\n ghp": ("https://h.example/p\r\n" + _GH, _GH), + "url \\n AKIA": ("https://console.aws.amazon.com/iam\n" + _AKIA, _AKIA), + "url \\n jwt": ("https://jwt.io\n" + _JWT_LONG, "FAKEsigFAKEsigFAKEsig"), + "url \\n sk-proj": ("https://platform.openai.com/api-keys\n" + _SKP, _SKP), + "url \\n token=": ("https://h.example/p\ntoken=Zq8wYv3Lp9Kd2xT", "Zq8wYv3Lp9Kd2xT"), + "url \\n DB_PASSWORD=": ( + "https://h.example/p\nDB_PASSWORD=Zq8wYv3Lp9Kd2xT", + "Zq8wYv3Lp9Kd2xT", + ), + "url \\n url \\n ghp": ("https://a.example/x\nhttps://b.example/y\n" + _GH, _GH), + "1b url \\n ghp": ( + "postgres://admin:P@ss/Xk9@db.example.com:5432/app\n" + _GH, + _GH, + ), + "past N \\n ghp": (_LONG_URL + "\n" + _GH, _GH), + "past N \\n token: \\n value": ( + _LONG_URL + "\ntoken:\nZq8wYv3Lp9Kd2xT", + "Zq8wYv3Lp9Kd2xT", + ), + "past N \\n ghp \\n url": (_LONG_URL + "\n" + _GH + "\nhttps://c.example/z", _GH), + # A URL on the key's own line: the cut there sends the key's line, masked at its start. + "past N \\n ghp/url": (_LONG_URL + "\n" + _GH + "/https://c.example/z", _GH), + "past N, many lines": ("https://h.example/x" + "\nabc" * 1100 + "\n" + _GH, _GH), + # cr3 M5: a key on the line after a slash, glued to a URL. + "past N /\\n AKIA glued to url": ( + _LONG_URL + "/\n" + _AKIA + "https://c.example/z", + _AKIA, + ), +} + + +@pytest.mark.parametrize("name", list(_LINE_STARTS)) +def test_a_key_at_a_line_start_in_a_one_url_reply_never_streams(name: str) -> None: + """cr2 H1: at 1- and 7-character deltas and as one delta, the stream equals cleaning the + whole text, and no event holds the secret's tail, in any case: a value cleaned as part of + a scheme comes out lower-cased (cr3 H1).""" + from fi.alk.providers.agent import _redact + + text, secret = _LINE_STARTS[name] + tail = secret[-12:].lower() + whole = _redact(text) + assert tail not in whole.lower() + for size in (1, 7, len(text)): + events = _events_of([text[i : i + size] for i in range(0, len(text), size)]) + assert "".join(events) == whole, size + assert not any(tail in event.lower() for event in events), size + + +# cr3 H1: past N before prose, a named value or a key on the line before a URL line. The +# branches find URL starts on the joined text, where that line is glued onto the URL's scheme; +# a cut there sent the name alone and cleaned the value as a scheme, lower-cased. Hawk's rows +# 1, 3 and 5. +_V = "Zq8wYv3Lp9Kd2xT" +_GRAFANA = "\n".join( + f"https://grafana.example.com/d/service-{i}/latency" for i in range(90) +) +_LINE_BEFORE_A_URL = { + "a long URL, DB_PASSWORD=V, a URL": ( + _LONG_URL + "\nDB_PASSWORD=" + _V + "\nhttps://b.example/y", + _V, + ), + "a long URL, token=V glued to a URL": ( + _LONG_URL + "\ntoken=" + _V + "https://c.example/z", + _V, + ), + "90 grafana URLs, GITHUB_TOKEN=ghp_, a URL": ( + _GRAFANA + "\nGITHUB_TOKEN=" + _GH + "\nhttps://github.com/org/repo", + _GH, + ), + # A name and its value on the next line go together. + "a long URL, token:, V, a URL": ( + _LONG_URL + "\ntoken:\n" + _V + "\nhttps://c.example/z", + _V, + ), + # A cut never splits a URL's userinfo from its host. + "a long user, :V@host on the next line, a URL": ( + "https://" + "u" * 4100 + "\n:" + _V + "@host.example/x\nhttps://c.example/z", + _V, + ), +} + + +@pytest.mark.parametrize("name", list(_LINE_BEFORE_A_URL)) +@pytest.mark.parametrize("size", [1, 4, 512, 0]) +def test_a_value_on_the_line_before_a_url_never_streams(name: str, size: int) -> None: + """cr3 H1: before prose, a shed cuts only at a line boundary, at 1-, 4- and 512-character + deltas and as one delta.""" + from fi.alk.providers.agent import _redact + + text, secret = _LINE_BEFORE_A_URL[name] + size = size or len(text) + tail = secret[-12:].lower() + whole = _redact(text) + assert tail not in whole.lower() + + events = _events_of([text[i : i + size] for i in range(0, len(text), size)]) + + assert "".join(events) == whole + assert not any(tail in event.lower() for event in events) + + +@pytest.mark.parametrize("size", [1, 7, 0]) +def test_a_line_start_masked_in_the_stream_is_logged_once( + size: int, caplog: pytest.LogCaptureFixture +) -> None: + """cr2 H1: the line-start mask's kinds go on the cleaner's one log line.""" + text = "https://github.com/settings/tokens\n" + _GH + size = size or len(text) + with caplog.at_level("WARNING"): + _events_of([text[i : i + size] for i in range(0, len(text), size)]) + + assert [ + r.getMessage() for r in caplog.records if r.name == "fi.alk.providers.agent" + ] == ["secrets redacted: kinds=github"] + + +@pytest.mark.parametrize("size", [1, 7, 0]) +def test_a_line_start_masked_by_a_shed_is_logged_once( + size: int, caplog: pytest.LogCaptureFixture +) -> None: + """cr3 L3: past N, the shed sends the key's line, masked, and logs its kind once.""" + text = _LONG_URL + "\n" + _GH + "/\nhttps://c.example/z" + size = size or len(text) + with caplog.at_level("WARNING"): + events = _events_of([text[i : i + size] for i in range(0, len(text), size)]) + + assert "[redacted]/" in events[0] + assert [ + r.getMessage() for r in caplog.records if r.name == "fi.alk.providers.agent" + ] == ["secrets redacted: kinds=github"] + + +# cr3 M6 and L1: an opening that fills N with scheme characters and no :// is prose, so the +# URL lines after it aren't glued onto it. L1's named limit ("an opening of 4,096 or more scheme +# characters with no ://") is closed by it, and its row now passes. +_LONG_OPENINGS = { + "M6: a query value on a later URL line": ( + "Zq8w" * 1100 + "\nhttps://registry.example.com/x\n&tok=" + _V, + _V, + ), + "L1: a key on the next line": ("a" * 4100 + "\n" + _GH, _GH), +} + + +@pytest.mark.parametrize("name", list(_LONG_OPENINGS)) +@pytest.mark.parametrize("size", [1, 7, 64, 0]) +def test_a_long_opening_with_no_scheme_streams_as_prose(name: str, size: int) -> None: + from fi.alk.providers.agent import _redact + + text, secret = _LONG_OPENINGS[name] + size = size or len(text) + tail = secret[-12:].lower() + whole = _redact(text) + assert tail not in whole.lower() + + events = _events_of([text[i : i + size] for i in range(0, len(text), size)]) + + assert "".join(events) == whole + assert not any(tail in event.lower() for event in events) + + +# cr2 M2: rule 1 across whitespace, then rule 2's next piece (S's r1r2.py). +_RULE1_THEN_RULE2 = [ + "See redis://:pa ss@h:6379\n?code=s3cr3tvalue99 then", + "See redis://u:pa ss@h.example:6379 &tok=s3cr3tvalue99 then", + "Try postgres://admin:Xk9/Qp+Zr\n 8@db.example.com:5432/app\n&tok=s3cr3tvalue99 ok", + # cr3 M2: the address piece's query ends in &, so rule 2 reaches the next piece. + "See redis://:pa ss@h:6379/?a=1& tok=s3cr3tvalue99 then", +] + + +@pytest.mark.parametrize("text", _RULE1_THEN_RULE2, ids=range(len(_RULE1_THEN_RULE2))) +def test_rule_2s_next_piece_holds_after_a_password_broken_by_whitespace( + text: str, +) -> None: + """cr2 M2: the piece a rule 1 match ends in is an address piece, at 1- and 5-character + deltas, and at every two-way split, where the @ arrives in the delta its piece ends in + (cr3 M2).""" + splits = [ + [text[i : i + size] for i in range(0, len(text), size)] for size in (1, 5) + ] + _every_split(text) + + assert _split_problems(text, splits) == [] + assert not any("s3cr3tvalue99" in event for event in _events_of(list(text))) + + +def test_a_password_broken_by_a_crlf_split_between_deltas_never_streams() -> None: + """cr2 M3: `\\r` and `\\n` in separate deltas, at 1-, 2- and 3-character deltas; and with + spaces and a tab before the `\\r` (cr3 M3).""" + for text in ( + "Try postgres://admin:Xk9/Qp+Zr\r\n8@db.example.com:5432/app now", + "Try postgres://admin:Xk9/Qp+Zr \t\r\n8@db.example.com:5432/app now", + ): + splits = [ + [text[i : i + size] for i in range(0, len(text), size)] + for size in (1, 2, 3) + ] + + assert _split_problems(text, splits) == [], text + + +def test_an_open_quoted_value_longer_than_the_hard_bound_still_streams() -> None: + """cr2 M4: a named value is held up to M (`_LONGEST_HELD_KEY`), not past it: text streams + before the message ends.""" + from fi.alk.harness.hosted_chat_entrypoint import _DeltaCleaner + + words = "The quick brown fox jumps over the lazy dog while the build runs. " + text = 'Set "password": "' + (words * 320)[: 20 * 1024] + cleaner = _DeltaCleaner() + + streamed = "".join(cleaner.feed(text[i : i + 4]) for i in range(0, len(text), 4)) + + assert len(streamed) > 1024 + + +def test_one_large_delta_of_passwords_broken_by_spaces_stays_linear() -> None: + """cr3 M1: `_address` finds its piece by bisection. One 256 KiB delta of rule 1 matches + that cross a space takes at most twice the same text with no broken password, best of 3, + interleaved (cr4 L1: an absolute time failed on a loaded machine).""" + import time + + from fi.alk.harness.hosted_chat_entrypoint import _DeltaCleaner + + def text(unit: str) -> str: + return (unit * (256 * 1024 // len(unit) + 1))[: 256 * 1024] + + broken, control = ( + text("See redis://:pa ss@h x y z "), + text("See redis://h:6379 x y z now "), + ) + best = {broken: 99.0, control: 99.0} + for _ in range(3): + for each in (broken, control): + cleaner = _DeltaCleaner() + start = time.perf_counter() + cleaner.feed(each) + best[each] = min(best[each], time.perf_counter() - start) + + assert best[broken] <= 2 * best[control], ( + f"{best[broken]:.3f} s, {best[control]:.3f} s" + ) + + +# -- TH-9500 change review r4 (cr4) ----------------------------------------------------------------- + +# cr4 H1: Dileep's rule 1b input with its password broken by a line break, after the / and +# before it, past N before prose. A cut between the password's lines sent its first half; the +# cut holds while the text after it, up to the next ://, may still be that URL's userinfo. +_RULE1B_BROKEN_PAST_N = { + "after the /": _LONG_URL + + "\npostgres://admin:Xk9/Qp+Zr\n8@db.example.com:5432/app\nhttps://c.example/z", + "before the /": _LONG_URL + + "\npostgres://admin:Xk\n9/Qp+Zr8@db.example.com:5432/app\nhttps://c.example/z", +} + + +@pytest.mark.parametrize("name", list(_RULE1B_BROKEN_PAST_N)) +@pytest.mark.parametrize("size", [1, 7, 512, 0]) +def test_a_rule_1b_password_broken_by_a_line_break_past_n_never_streams( + name: str, size: int +) -> None: + from fi.alk.providers.agent import _redact + + text = _RULE1B_BROKEN_PAST_N[name] + size = size or len(text) + whole = _redact(text) + assert "qp+zr" not in whole.lower() + + events = _events_of([text[i : i + size] for i in range(0, len(text), size)]) + + assert "".join(events) == whole + assert not any("qp+zr" in event.lower() for event in events) + + +# cr4 M1: honest replies that are only URLs, one per line. Each line ends in scheme characters, +# so joined it is glued onto the next line's scheme; the cut must still fall between them. +_URL_LISTS = { + "400 grafana dashboards": "\n".join( + f"https://grafana.example.com/d/service-{i}/latency" for i in range(400) + ), + "800 bare hosts": "\n".join(f"https://host{i}.example.com" for i in range(800)), + "400 GitHub repositories": "\n".join( + f"https://github.com/org-{i}/repo-{i}" for i in range(400) + ), +} + + +@pytest.mark.parametrize("name", list(_URL_LISTS)) +@pytest.mark.parametrize("size", [7, 512, 0]) +def test_an_honest_url_list_streams_before_its_end(name: str, size: int) -> None: + """cr4 M1: the stream equals the whole text, and most of it goes before the flush.""" + from fi.alk.harness.hosted_chat_entrypoint import _DeltaCleaner + from fi.alk.providers.agent import _redact + + text = _URL_LISTS[name] + size = size or len(text) + cleaner = _DeltaCleaner() + + streamed = "".join( + cleaner.feed(text[i : i + size]) for i in range(0, len(text), size) + ) + + assert streamed + cleaner.flush() == _redact(text) + assert len(streamed) > len(text) // 2 + + +def _best_interleaved(texts: dict[str, str], size: int = 4) -> dict[str, float]: + """Each text fed in `size`-character deltas, best of 3, the texts taking turns so that a + loaded machine slows them alike.""" + import time + + from fi.alk.harness.hosted_chat_entrypoint import _DeltaCleaner + + best = dict.fromkeys(texts, 99.0) + for _ in range(3): + for name, text in texts.items(): + cleaner = _DeltaCleaner() + start = time.perf_counter() + for i in range(0, len(text), size): + cleaner.feed(text[i : i + size]) + cleaner.flush() + best[name] = min(best[name], time.perf_counter() - start) + return best + + +# 16 KiB held in 4-character deltas costs the same for any text it holds, a cut that is walked +# once or one that is walked again on every delta aside; one URL with no breaks is the control. +_HELD_16K = 16 * 1024 +_HELD_CONTROL = ("https://a.example/" + "x" * _HELD_16K)[:_HELD_16K] + + +def test_a_held_cut_is_not_walked_again_on_every_delta() -> None: + """cr4 M2: while a cut holds, the held text grows by lines it can't cut at; they are walked + once, not on every delta. R's shape at 16 KiB in 4-character deltas takes at most twice a + control (walked on every delta, about a minute).""" + held = ("https://hostA\nhttps://hostB" + "\nx" * _HELD_16K)[:_HELD_16K] + # The cut held at a line that is only a URL's own `://` keeps its place too. + at_scheme = ("https://hostA/a\nhttps\n://hostB/p\nhttps://c/" + "\nx" * _HELD_16K)[:_HELD_16K] + + best = _best_interleaved({"held": held, "at_scheme": at_scheme, "control": _HELD_CONTROL}) + + assert best["held"] <= 2 * best["control"], best + assert best["at_scheme"] <= 2 * best["control"], best + + +def test_a_cut_the_userinfo_guard_holds_is_not_walked_again_on_every_delta() -> None: + """cr4b M2: the other way a cut is held: an @ line before the next URL. The guard path + keeps the held cut too.""" + held = ( + "https://a.example/\nalice@example.com\nhttps://b.example/" + "\nx" * _HELD_16K + )[:_HELD_16K] + + best = _best_interleaved({"held": held, "control": _HELD_CONTROL}) + + assert best["held"] <= 2 * best["control"], best + + +@pytest.mark.parametrize("size", [7, 512, 0]) +def test_a_cut_held_by_an_at_line_is_released_by_the_url_lines_after_it( + size: int, +) -> None: + """cr4b M2: a long URL, an email line, then 800 URL lines (24,844 joined). The cut the + @ line holds moves on with the URL starts after it: the list streams as stored, not as + `[redacted]`, at every size (a held cut that stays put past a later start fails at 7).""" + from fi.alk.providers.agent import _redact + + text = ( + _LONG_URL + + "\nalice@example.com\nhttps://c.example/z" + + "".join(f"\nhttps://h{i}.example.com/p" for i in range(800)) + ) + step = size or len(text) + + events = _events_of([text[i : i + step] for i in range(0, len(text), step)]) + + assert "".join(events).strip() == _redact(text.strip()) + assert sum(len(event) for event in events[:-1]) > len(text) // 2 + + +def test_tab_split_lines_are_cut_without_one_call_blocking() -> None: + """cr4b H3: a URL whose lines are split by tabs, 1 MiB of them, then a :// that moves the + cut. No call takes a second (a search back to the held text's start for every line took + 2 s at 1 MiB, 9 s at 2), and the held text, breaks counted, stays within the hard bound.""" + import time + + from fi.alk.harness.hosted_chat_entrypoint import _HOLD_HARD, _DeltaCleaner + + run = 128 + text = ( + "https://a.example/@x" + + ("\t" * run + "x") * (1024 * 1024 // (run + 1)) + + "://b" + ) + cleaner = _DeltaCleaner() + slowest = most = 0 + for i in range(0, len(text), 4096): + start = time.perf_counter() + cleaner.feed(text[i : i + 4096]) + slowest = max(slowest, time.perf_counter() - start) + most = max(most, len(cleaner.held)) + cleaner.flush() + + assert slowest < 1.0, f"{slowest:.2f} s" + assert most <= _HOLD_HARD + 4096 + + +def test_the_walk_over_tab_split_lines_costs_each_line_its_own_length() -> None: + """cr4b H3: `_line_cut` on 1 MiB of tab-split lines, with the cut on the first, takes at + most twice the same on newline-split lines (searching back for every line took 2 s).""" + import time + + from fi.alk.harness.hosted_chat_entrypoint import _URL_BREAKS, _DeltaCleaner + + run = 128 + unit = ("\t" * run + "x", "\n" * run + "x") + best = [99.0, 99.0] + for _ in range(3): + for n, each in enumerate(unit): + raw = "https://a.example/@x" + each * (1024 * 1024 // (run + 1)) + "://b" + start = time.perf_counter() + _DeltaCleaner()._line_cut(raw, raw.translate(_URL_BREAKS), 0) + best[n] = min(best[n], time.perf_counter() - start) + + assert best[0] <= 2 * best[1] + 0.05, best + + +def test_a_name_its_equals_and_its_value_on_three_lines_past_n_is_a_named_limit() -> ( + None +): + """cr4 M3, a named limit: before prose and past N, a name, its `=` and its value each on a + line of its own, then a URL line. No cut keeps all three lines with the name: the value + streams joined onto the URL's scheme, lower-cased, where the whole text masks it. The + stream is pinned as it is, at every size.""" + text = _LONG_URL + "\ntoken\t=\tZq8wYv3Lp9Kd2xT\nhttps://c.example/z" + for size in (1, 7, 512, len(text)): + events = _events_of([text[i : i + size] for i in range(0, len(text), size)]) + assert "".join(events) == ( + _LONG_URL + "token=zq8wyv3lp9kd2xthttps://c.example/z" + ), size + + +# -- TH-9500 change review r5 (cr5): the offset sweeps as tests ------------------------------------- + +# Three rounds of hand-picked rows each passed and then failed the next offset class a sweep +# found. So each class is swept here: a break at every offset after the ://, or at every place +# the input has one, and the rule is the one chat has: the stream, joined, is the stored message +# (`_redact(reply.strip())`), and no event holds the secret, compared lower-cased. A sweep that +# ran fewer cases than it should have, or whose check was taken out, fails on its count. +_SWEEP_BREAKS = {"n": "\n", "t": "\t", "rn": "\r\n"} +_SWEEP_GRAFANA = "\n".join( + f"https://grafana.example.com/d/service-{i}/latency" for i in range(90) +) +# (url, the tail of the password that must never stream) +_SWEEP_URLS = { + "1b": ("postgres://admin:Xk9/Qp+Zr8@db.example.com:5432/app", "qp+zr"), + "r1": ("postgres://admin:S3cretPw9x@db.example.com/app", "cretp"), + "r1-nouser": ("redis://:S3cretPw9x@cache.internal:6379/0", "cretp"), + "1b-at": ("postgres://admin:P@ss/Xk9@db.example.com:5432/app", "ss/xk"), + "1b-ash": ("redis://:p@w/x9@cache.internal:6379/0", "w/x9"), + "1b-pass": ("postgres://admin:Pass@w/x9@db.example.com/app", "w/x9"), +} +_SWEEP_TAILS = ("\nhttps://c.example/z", "https://c.example/z") + + +def _swept(cases, size: int) -> tuple[int, list[str], int]: + """(cases held, the ones that were not, cases skipped) over `cases` of (label, text, + secret): each streamed in deltas of `size`. A case whose stored message keeps the secret is + skipped: that is the base cleaner's, not the stream's.""" + from fi.alk.providers.agent import _redact + + held, bad, skipped = 0, [], 0 + for label, text, secret in cases: + whole = _redact(text.strip()) + if secret in whole.lower(): + skipped += 1 + continue + events = _events_of([text[i : i + size] for i in range(0, len(text), size)]) + joined = "".join(events) + leak = any(secret in event.lower() for event in events) + if leak or joined.strip() != whole: + bad.append(f"{label}: {'leak' if leak else 'unequal'}") + else: + held += 1 + return held, bad, skipped + + +def _assert_swept(cases, size: int, at_least: int) -> None: + held, bad, skipped = _swept(cases, size) + assert not bad, f"{len(bad)} of {held + len(bad)}: {bad[:6]}" + # Every case is held or bad, so an emptied sweep, or a check taken out, changes this. + assert held >= at_least, f"swept {held}, skipped {skipped}" + + +def test_a_sweep_reports_a_leak_and_an_unequal_stream(monkeypatch) -> None: + """The sweeps' own check, on streams made to fail it: a sweep that cannot see a leak or a + difference would pass on any tree.""" + import sys + + here = sys.modules[__name__] + case = [("c", "See https://h.example/p now", "zzzzz")] + + monkeypatch.setattr( + here, "_events_of", lambda deltas: ["See https://h.example/p now"] + ) + assert _swept(case, 5) == (1, [], 0) + monkeypatch.setattr( + here, "_events_of", lambda deltas: ["See https://h.example/p ZZZZZ"] + ) + assert _swept(case, 5) == (0, ["c: leak"], 0) + monkeypatch.setattr( + here, "_events_of", lambda deltas: ["See https://h.example/q now"] + ) + assert _swept(case, 5) == (0, ["c: unequal"], 0) + # The base cleaner's own miss is no stream's: skipped, not held. + assert _swept([("k", "See zzzzz now", "zzzzz")], 5) == (0, [], 1) + + +def _url_sweep(size: int, brk: str): + """A credentialed URL after a long URL, with the break at every offset after its ://.""" + # One delta per character over the long URL alone keeps the 1-character run short. + prefixes = (_LONG_URL,) if size == 1 else (_LONG_URL, _SWEEP_GRAFANA) + tails = _SWEEP_TAILS[:1] if size == 1 else _SWEEP_TAILS + for pn, prefix in enumerate(prefixes): + for name, (url, secret) in _SWEEP_URLS.items(): + for k in range(url.index("://") + 3, len(url) + 1): + for tn, tail in enumerate(tails): + text = prefix + "\n" + url[:k] + brk + url[k:] + tail + yield f"{name}/{pn}/{k}/{tn}", text, secret + + +@pytest.mark.parametrize("brk", list(_SWEEP_BREAKS)) +@pytest.mark.parametrize("size", [1, 7, 64, 512]) +def test_a_credentialed_url_broken_at_any_offset_past_n_streams_as_stored( + size: int, brk: str +) -> None: + """cr4b H1, H2: U's sweep. After a long URL (or a URL list), a credentialed URL with a + break at every offset from just after its :// to its end, before and after its @, inside its + host, and before its port.""" + _assert_swept(_url_sweep(size, _SWEEP_BREAKS[brk]), size, 215 if size == 1 else 860) + + +def _rule_1b_sweep(brk: str): + """R's: a rule 1b password (it holds a /) broken after each of its characters.""" + for pn, prefix in enumerate((_LONG_URL, _SWEEP_GRAFANA)): + for password in ("Xk9/Qp+Zr8", "Xk9/Qp+Zr8Wv5Ty", "ab/cdEFgh12"): + for k in range(1, len(password) + 1): + for tn, tail in enumerate( + ("\nhttps://c.example/z", "https://c.example/z") + ): + text = ( + prefix + + "\npostgres://admin:" + + password[:k] + + brk + + password[k:] + + "@db.example.com:5432/app" + + tail + ) + yield ( + f"{pn}/{password}/{k}/{tn}", + text, + password.split("/")[1][:5].lower(), + ) + + +@pytest.mark.parametrize("brk", list(_SWEEP_BREAKS)) +@pytest.mark.parametrize("size", [1, 7, 64, 512]) +def test_a_rule_1b_password_broken_at_any_offset_past_n_streams_as_stored( + size: int, brk: str +) -> None: + """cr4 H1: R's sweep, a break after each character of a password that holds a /.""" + _assert_swept(_rule_1b_sweep(_SWEEP_BREAKS[brk]), size, 144) + + +def _dodge_cases(brk: str): + """Every :// held is broken by a line break, so the guard that looks for one finds none + (cr4b H2), and rule 1b passwords broken after the @ or inside the host (cr4b H1).""" + long = _LONG_URL + "/" + z = "https://c.example/" + "z" * 4100 + cases = { + "a :/ break /": ( + long + "\npostgres:/\n/admin:Xk9/Qp+Zr\n8@db.example.com:5432/app\n" + z, + "qp+zr", + ), + "b : break //": ( + long + "\npostgres:\n//admin:Xk9/Qp+Zr\n8@db.example.com:5432/app\n" + z, + "qp+zr", + ), + "c :/ break /, P@ss/Xk9@": ( + long + "\npostgres:/\n/admin:P@ss/Xk9@\ndb.example.com:5432/app\n" + z, + "ss/xk9", + ), + "d pw broken twice": ( + long + "\npostgres:/\n/admin:Xk9/Qp\n+Zr\n8@db.example.com:5432/app\n" + z, + "qp+zr", + ), + "e control, :// whole": ( + long + "\npostgres://admin:Xk9/Qp+Zr\n8@db.example.com:5432/app\n" + z, + "qp+zr", + ), + } + long_url = _LONG_URL + other = "https://c.example/z" + h123 = { + "H1 broken after the path's @": ( + long_url + + "\npostgres://admin:P@ss/Xk9@\ndb.example.com:5432/app\n" + + other, + "ss/xk9", + ), + "H1 addendum shape, port in password": ( + long_url + "\npostgres://admin:P@ss:99/Xk9@\ndb.example.com/app\n" + other, + "ss:99/xk9", + ), + "H1 redis 1b, glued URL after": ( + long_url + "\nredis://:p@w/x9@\ncache.internal:6379/0" + other, + "w/x9", + ), + "H1 broken inside the host": ( + long_url + + "\npostgres://admin:P@ss/Xk9@db.\nexample.com:5432/app\n" + + other, + "ss/xk9", + ), + "H1 control, under N": ( + "https://h.example/p\npostgres://admin:P@ss/Xk9@\ndb.example.com:5432/app\n" + + other, + "ss/xk9", + ), + "H2 :/ break /, password broken": ( + long_url + + "/\npostgres:/\n/admin:Xk9/Qp+Zr\n8@db.example.com:5432/app\nhttps://c.example/" + + "z" * 4100, + "qp+zr", + ), + "H2 control, :// whole": ( + long_url + + "/\npostgres://admin:Xk9/Qp+Zr\n8@db.example.com:5432/app\nhttps://c.example/" + + "z" * 4100, + "qp+zr", + ), + } + for name, (text, secret) in {**cases, **h123}.items(): + yield name, text.replace("\n", brk), secret + + +@pytest.mark.parametrize("brk", list(_SWEEP_BREAKS)) +@pytest.mark.parametrize("size", [1, 7, 64, 512]) +def test_a_url_whose_own_scheme_or_host_is_broken_past_n_streams_as_stored( + size: int, brk: str +) -> None: + """cr4b H1, H2: the guard may not depend on a :// being whole in the text held, nor on the + break falling in a place that leaves the password's host on the next line.""" + _assert_swept(_dodge_cases(_SWEEP_BREAKS[brk]), size, 12) + + +def _honest_lists() -> dict[str, str]: + def alt(n: int, a: str, b: str) -> str: + return "\n".join(a.format(i=i) + "\n" + b.format(i=i) for i in range(n)) + + return { + "github + email x300": alt( + 300, "https://github.com/user{i}", "user{i}@example.com" + ), + "email + github x300": "https://example.com/team\n" + + "\n".join( + f"user{i}@example.com\nhttps://github.com/user{i}" for i in range(300) + ), + "url + git@ x300": alt( + 300, + "https://gitlab.example.com/org/repo{i}", + "git@gitlab.example.com:org/repo{i}.git", + ), + "url + mailto x300": alt( + 300, "https://example.com/people/p{i}", "mailto:p{i}@example.com" + ), + "url + @mention x400": alt( + 400, "https://github.com/org/repo{i}/pull/{i}", "@reviewer{i}" + ), + "url + img@sha256 x150": alt( + 150, + "https://hub.docker.com/r/org/img{i}", + "org/img{i}@sha256:" + "ab12" * 16, + ), + "medium /@user x500": "\n".join( + f"https://medium.com/@writer{i}" for i in range(500) + ), + "npm /@scope/pkg x500": "\n".join( + f"https://www.npmjs.com/package/@scope{i}/pkg" for i in range(500) + ), + "control: github only x600": "\n".join( + f"https://github.com/user{i}" for i in range(600) + ), + } + + +# The lists that stream: an @ in a URL's own path or a /@ segment is no userinfo. +_STREAMING_LISTS = ( + "medium /@user x500", + "npm /@scope/pkg x500", + "control: github only x600", +) +# cr4b M1, a named limit: an @ on the line before a URL could be the end of a userinfo, and the +# cut after it can't tell, so the held text isn't cut while a later URL follows an @ line. Past +# _HOLD_HARD it is sent as `[redacted]` and the rest of the token is dropped; short of it, +# nothing goes before the flush. (Holding only while rule 1 finds a userinfo that would cross +# the cut was tried: rule 1 is the cross-whitespace rule, and the plain userinfo of a URL on one +# line isn't its span, so it let passwords through.) The raw text is counted toward the bound, +# breaks too, so a list of about 16,000 joined characters with a break per URL is past it. +_AT_LINE_LISTS_REDACTED = ( + "url + git@ x300", + "url + @mention x400", + "url + img@sha256 x150", + "url + mailto x300", +) +_AT_LINE_LISTS_HELD = ("github + email x300", "email + github x300") + + +def _fed(text: str, size: int) -> tuple[str, str]: + """What the stream sends before the flush, and what the flush sends.""" + from fi.alk.harness.hosted_chat_entrypoint import _DeltaCleaner + + cleaner = _DeltaCleaner() + sent = "".join(cleaner.feed(text[i : i + size]) for i in range(0, len(text), size)) + return sent, cleaner.flush() + + +@pytest.mark.parametrize("size", [7, 512, 0]) +def test_an_honest_url_list_streams_as_stored_and_before_its_end(size: int) -> None: + """URL-only lists past _HOLD_SOFT joined with an @ in a path: the stream is the stored + message, and most of it goes before the flush.""" + from fi.alk.providers.agent import _redact + + lists = _honest_lists() + for name in _STREAMING_LISTS: + text = lists[name] + sent, flushed = _fed(text, size or len(text)) + assert (sent + flushed).strip() == _redact(text.strip()), name + assert len(sent) > len(text) // 2, name + assert ( + len(_STREAMING_LISTS) + len(_AT_LINE_LISTS_REDACTED) + len(_AT_LINE_LISTS_HELD) + == 9 + ) + assert len(lists) == 9 + + +@pytest.mark.parametrize("size", [7, 512, 0]) +def test_an_honest_url_list_with_an_at_line_before_each_url_is_a_named_limit( + size: int, +) -> None: + """cr4b M1, a named limit: pinned as it is, so that a change to it shows.""" + from fi.alk.providers.agent import _redact + + lists = _honest_lists() + for name in _AT_LINE_LISTS_REDACTED + _AT_LINE_LISTS_HELD: + text = lists[name] + sent, flushed = _fed(text, size or len(text)) + if name in _AT_LINE_LISTS_REDACTED: + assert sent + flushed == "[redacted]", name + else: + assert sent == "", name + assert flushed.strip() == _redact(text.strip()), name + + +# -- TH-9500 change review r5, H1: a break just before a credentialed URL's own :// ----------------- +_PRE_SCHEME_URLS = { + "1b": ("postgres://admin:Xk9/Qp+Zr8@db.example.com:5432/app", "qp+zr"), + "1b-at": ("postgres://admin:P@ss/Xk9@db.example.com:5432/app", "ss/xk"), + "1b-port": ("postgres://admin:P@ss:99/Xk9@db.example.com/app", "ss:99"), + "1b-ash": ("redis://:p@w/x9@cache.internal:6379/0", "w/x9"), + "r1": ("postgres://admin:S3cretPw9x@db.example.com/app", "cretp"), +} + + +def _pre_scheme_cases(): + for name, (url, secret) in _PRE_SCHEME_URLS.items(): + at = url.index("://") + for kind, brk in _SWEEP_BREAKS.items(): + text = _LONG_URL + "\n" + url[:at] + brk + url[at:] + "\nhttps://c.example/z" + yield f"{name}/{kind}", text, secret + + +@pytest.mark.parametrize("size", [1, 7, 64, 512, 0], ids=lambda size: f"size_{size or 'whole'}") +def test_a_credentialed_url_broken_just_before_its_scheme_past_n_streams_as_stored(size) -> None: + """Past N before prose, the line cut can fall between a URL's scheme and its own ://, where + the text after the cut still holds the userinfo. Each of 5 URLs × 3 breaks streams equal to + the stored message, with no part of its password in any event.""" + cases = list(_pre_scheme_cases()) + assert len(cases) == 15 + if size == 0: + held = sum(_swept([case], len(case[1]))[0] for case in cases) + bad = [b for case in cases for b in _swept([case], len(case[1]))[1]] + assert not bad, bad + assert held == len(cases) + return + _assert_swept(cases, size, at_least=len(cases)) + + +@pytest.mark.parametrize("size", [1, 7, 64]) +def test_a_scheme_line_then_a_lone_colon_line_past_n_streams_as_stored(size) -> None: + """The scheme, its `:` and its `//` on three lines, with a password holding `://` and an @ + on a line of its own: the cut must not fall between the scheme and its `://`.""" + text = ( + _LONG_URL + + "\npostgres\n:\n//admin:S3cretPw9x/a://b\n@\ndb.example.com/app\nhttps://c.example/z" + ) + _assert_swept([("scheme, :, //", text, "s3cretpw9x")], size, at_least=1) diff --git a/tests/harness/test_contract_writer.py b/tests/harness/test_contract_writer.py index 2bcfa3e9a..9b6da78ac 100644 --- a/tests/harness/test_contract_writer.py +++ b/tests/harness/test_contract_writer.py @@ -494,7 +494,7 @@ def test_one_save_logs_one_line_however_many_urls_are_refused( logged = [record for record in caplog.records if record.name == "fi.alk.providers.agent"] assert [record.getMessage() for record in logged] == [ - "a URL was redacted whole: class=ValueError" + "secrets redacted: url=ValueError" ] assert PASSWORD not in caplog.text assert PASSWORD not in (tmp_path / "contract.json").read_text(encoding="utf-8") diff --git a/tests/harness/test_hosted_entrypoint.py b/tests/harness/test_hosted_entrypoint.py index 7af13a83e..8245df3ae 100644 --- a/tests/harness/test_hosted_entrypoint.py +++ b/tests/harness/test_hosted_entrypoint.py @@ -5495,6 +5495,26 @@ async def reading(_args): assert "hunter2" not in events and "sk-1" not in events +def test_a_tool_result_is_cleaned_before_it_is_cut_at_600_characters(): + """TH-9500 X1 (plan v1.1 section 6, r1 M1): session.py:388 cleans before it cuts, so a key + past character 597 is masked whole, and a query URL before it can't move the cut rules.""" + from fi.alk.harness.backends.base import SessionSpec, ToolReturned + from fi.alk.harness.session import RESULT, Stage, Turn + + key = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8" + # Cut first, 6 of the key's characters would be left before the "...": too few for the + # rule that masks a key cut short. + text = "see https://h.example/x?a=1 then " + "y " * 277 + key + assert text.index(key) + len("ghp_A1b2C3") == 597 + stage = Stage(SessionSpec(system_prompt=""), name="understand") + + [event] = stage._events(ToolReturned(id="c1", text=text), Turn()) + + assert event.kind == RESULT + assert "A1b2C3" not in event.text + assert event.text == "see https://h.example/x then " + "y " * 277 + "[redacted]" + + def test_the_activity_printed_to_stdout_loses_url_credentials(capsys, monkeypatch): """Change review r1, A-H1: the platform keeps the tail of the guest's stdout.""" from fi.alk.harness import cli diff --git a/tests/harness/test_outbound.py b/tests/harness/test_outbound.py index 92294b857..0dd3b0726 100644 --- a/tests/harness/test_outbound.py +++ b/tests/harness/test_outbound.py @@ -3400,7 +3400,7 @@ def test_build_event_record_redacts_a_dsn_in_world_unhealthy_cause() -> None: ) cause = record["payload"]["cause"] assert "s3cr3t" not in cause - assert cause == "connect failed: postgresql://harness:***@localhost:14000/w0" + assert cause == "connect failed: postgresql://localhost:14000/w0" # The digest must match the REDACTED bytes actually spooled, not the original with the secret. assert record["digest"] == event_payload_digest(record["payload"]) @@ -3423,7 +3423,7 @@ def test_build_event_record_redacts_terminal_failure_message() -> None: ) message = record["payload"]["failure"]["message"] assert "hunter2" not in message - assert "postgresql://harness:***@localhost:14000/w0" in message + assert "postgresql://localhost:14000/w0" in message def test_build_result_receipt_redacts_sub_goal_and_evaluation_reasons_and_failure_message() -> ( @@ -3463,11 +3463,9 @@ def test_build_result_receipt_redacts_sub_goal_and_evaluation_reasons_and_failur }, ) assert "pw123" not in str(receipt) - assert ( - receipt["sub_goals"][0]["reason"] == "leaked postgresql://harness:***@host/db" - ) - assert receipt["evaluations"][0]["reason"] == "saw postgresql://harness:***@host/db" - assert receipt["failure"]["message"] == "postgresql://harness:***@host/db" + assert receipt["sub_goals"][0]["reason"] == "leaked postgresql://host/db" + assert receipt["evaluations"][0]["reason"] == "saw postgresql://host/db" + assert receipt["failure"]["message"] == "postgresql://host/db" # The digest must match the redacted bytes. ResultReceiptDraft.model_validate(receipt) # round-trips cleanly, digest agrees @@ -3475,7 +3473,7 @@ def test_build_result_receipt_redacts_sub_goal_and_evaluation_reasons_and_failur def test_redact_outbound_text_scrubs_userinfo_and_extra_secrets() -> None: assert ( redact_outbound_text("postgresql://harness:pw@localhost/db") - == "postgresql://harness:***@localhost/db" + == "postgresql://localhost/db" ) assert redact_outbound_text("no secrets here") == "no secrets here" assert ( @@ -3498,12 +3496,12 @@ def test_redact_outbound_text_scrubs_userinfo_and_extra_secrets() -> None: [ # P3: the empty-username shape -- the canonical Redis/RabbitMQ/Mongo DSN form -- used to # leave the password verbatim (the old pattern required a NON-empty username group). - ("redis://:secretpw@h:6379/0", "redis://:***@h:6379/0"), + ("redis://:secretpw@h:6379/0", "redis://h:6379/0"), # A password containing '@' must not truncate the mask at the first '@' seen. - ("mysql://u:p@ss@h/db", "mysql://u:***@h/db"), + ("mysql://u:p@ss@h/db", "mysql://h/db"), # A bare token/username-only userinfo (no ':') is the standard shape a bearer token takes # in git/registry output -- must be masked outright, not treated as "just a username." - ("https://ghp_TOKEN@github.com/o/r.git", "https://***@github.com/o/r.git"), + ("https://ghp_TOKEN@github.com/o/r.git", "https://github.com/o/r.git"), # Negative: a path segment containing '@' with no userinfo before it must be left alone. ("https://example.com/a@b", "https://example.com/a@b"), # Negative: no "://" at all -- not a userinfo shape. @@ -3516,6 +3514,73 @@ def test_redact_outbound_text_handles_the_empty_username_and_bare_token_userinfo assert redact_outbound_text(raw) == expected +# -- TH-9500: the kit's one cleaner runs here too (plan v1.1 section 4, test O1) ---------------- + +_FAKE_KEY = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8" + + +def test_a_declared_secret_is_matched_whole_before_the_cleaner_can_mask_part_of_it() -> ( + None +): + """The order: extra values first. A declared secret holding a word and a key would lose + only its key to the cleaner, and the rest would no longer match it exactly (r1 M5).""" + declared = f"deploy {_FAKE_KEY}" + + out = redact_outbound_text(f"ran {declared} ok", extra_secret_values=(declared,)) + + assert out == "ran *** ok" + assert "deploy" not in out + + +def test_a_key_and_a_broken_password_are_masked_with_one_log_line( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.DEBUG): + out = redact_outbound_text( + f"push with {_FAKE_KEY} to https://bot:hun ter2@h.example/v1 now" + ) + + assert out == "push with [redacted] to https://h.example/v1 now" + assert [ + r.getMessage() for r in caplog.records if r.name == "fi.alk.providers.agent" + ] == ["secrets redacted: kinds=github,rule1"] + + +def test_nothing_masked_logs_nothing(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.DEBUG): + assert ( + redact_outbound_text("token: required, see docs") + == "token: required, see docs" + ) + + assert [r for r in caplog.records if r.name == "fi.alk.providers.agent"] == [] + + +# Rule 1b (design addendum v2.5.2): Dileep's #278 threads P1 and P3, and Ash's two inputs. +_RULE1B_ROWS = { + "postgres://admin:Xk9/Qp+Zr8@db.example.com:5432/app": ( + "postgres://db.example.com:5432/app" + ), + "https://user:pa#ss@host.example/x": "https://host.example/x", + "postgres://admin:P@ss/Xk9@db.example.com:5432/app": ( + "postgres://db.example.com:5432/app" + ), + "smtp://me@gmail.com:app/pass@smtp.gmail.com:587": "smtp://smtp.gmail.com:587", +} + + +@pytest.mark.parametrize("raw", list(_RULE1B_ROWS), ids=range(len(_RULE1B_ROWS))) +def test_a_password_holding_an_at_sign_or_a_slash_leaves_no_part_behind( + raw: str, +) -> None: + expected = _RULE1B_ROWS[raw] + + assert redact_outbound_text(raw) == expected + assert redact_outbound_text(f"failed on {raw} twice") == ( + f"failed on {expected} twice" + ) + + # --- ArtifactsClient ----------------------------------------------------------------------------- diff --git a/tests/harness/test_reading_entry.py b/tests/harness/test_reading_entry.py index a5a062076..262b70bc9 100644 --- a/tests/harness/test_reading_entry.py +++ b/tests/harness/test_reading_entry.py @@ -550,7 +550,7 @@ def crash(*_args): "retryable": True, } assert failure["message"].startswith( - "RuntimeError: could not fetch https://user:***@host/repo.git " + "RuntimeError: could not fetch https://host/repo.git " ) assert "pw" not in failure["message"] assert MESSAGE_LIMIT == 2000 diff --git a/tests/harness/test_scenario_source_provision_agent.py b/tests/harness/test_scenario_source_provision_agent.py index 03f820a69..352eb8066 100644 --- a/tests/harness/test_scenario_source_provision_agent.py +++ b/tests/harness/test_scenario_source_provision_agent.py @@ -143,7 +143,7 @@ def test_a_url_urlsplit_refuses_is_registered_redacted_and_logged_by_class_only( # Once for the document and once for the prompt. logged = [record for record in caplog.records if record.name == "fi.alk.providers.agent"] assert [record.getMessage() for record in logged] == [ - "a URL was redacted whole: class=ValueError" + "secrets redacted: url=ValueError" ] * 2 assert all(record.exc_info is None for record in logged) assert "hunter2" not in caplog.text @@ -452,3 +452,210 @@ def test_the_document_and_the_reading_hold_the_same_cleaned_prompt(tmp_path: Pat assert payload["agent"]["agent"]["behaviour"][0]["instructions"] == cleaned assert reading["agent"]["behaviour"][0]["instructions"] == cleaned assert "hunter2" not in json.dumps(reading) and "sk-1" not in json.dumps(reading) + + +# -- TH-9500: the persona record is cleaned before it leaves (plan v1.1 section 5, test P1) ------ + +_FAKE_KEY = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8" + + +def test_a_key_in_each_cleaned_persona_field_is_masked_and_the_rest_sent_as_written() -> ( + None +): + body = { + "name": f"Dana {_FAKE_KEY}", + "use_case": f"Refund {_FAKE_KEY}", + "instruction": f"She pastes {_FAKE_KEY} into the chat", + "tests": "The agent refuses password=Zq8wYv3Lp9 politely", + "persona": { + "role": f"admin {_FAKE_KEY}", + "background": f"keeps {_FAKE_KEY} in a note", + "secret_motive": "wants a refund", + "traits": ["calm", f"uses {_FAKE_KEY}"], + }, + } + scenario = SimpleNamespace( + presented=ss._presented(body, scenario_key="refund-ghp_key-s1"), + presented_fields=ss._presented( + body, scenario_key="refund-ghp_key-s1", fields=True + ), + scenario_key="refund-ghp_key-s1", + ) + + payload = ss._provision_payload( + "Run ghp_x", [scenario], ["task_completion"], "", "chat", agent_name="Desk" + ) + + [persona] = payload["personas"] + assert _FAKE_KEY not in json.dumps(payload) + assert "Zq8wYv3Lp9" not in json.dumps(payload) + assert persona == { + "name": "Dana [redacted]", + "scenario_name": "Refund [redacted]", + "role": "admin [redacted]", + "situation": "She pastes [redacted] into the chat", + "outcome": "The agent refuses password=[redacted] politely", + "persona": { + "role": "admin [redacted]", + # The key markers apply to the persona's own keys too. + "background": "keeps [redacted] in a note", + "secret_motive": "[redacted]", + "traits": ["calm", "uses [redacted]"], + }, + "scenario_key": "refund-ghp_key-s1", + } + # As written: the platform matches on scenario_key, and the rest is the kit's own. + assert payload["name"] == "Run ghp_x" + assert payload["agent_name"] == "Desk" + assert payload["chosen_evals"] == ["task_completion"] + assert payload["modality"] == "chat" + # The judge's record is left as it was. + assert scenario.presented["name"] == f"Dana {_FAKE_KEY}" + + +def _sent(body: dict[str, Any], scenario_key: str = "s1") -> tuple[dict, dict]: + """The payload's persona for a scenario built from `body` as `_load_one` builds it, and the + record the judge reads.""" + presented = ss._presented(body, scenario_key=scenario_key) + scenario = SimpleNamespace( + presented=presented, + presented_fields=ss._presented(body, scenario_key=scenario_key, fields=True), + scenario_key=scenario_key, + ) + [persona] = ss._provision_payload("Run", [scenario], (), "", "chat")["personas"] + return persona, presented + + +def _straddling(at: int) -> str: + text = ("Caller says: " + "please help me with the refund " * 10)[:at] + text += " " + _FAKE_KEY + " and then hangs up politely." + assert text.index(_FAKE_KEY) == at + 1 + return text + + +def _masked(text: str) -> str: + at = text.index(_FAKE_KEY) + return text[:at] + "[redacted]" + text[at + len(_FAKE_KEY) :] + + +@pytest.mark.parametrize("at", [222, 244]) +@pytest.mark.parametrize("where", ["persona", "name_falls_back_to_the_scenario"]) +def test_a_key_straddling_the_255_character_cut_of_name_and_role_is_masked( + at: int, where: str +) -> None: + """P1 (cr1 M4, cr2 M1): `persona_of` cuts `name` and `role` at 255 characters, so the record + sent is `persona_of` over cleaned fields, including a name that falls back to the scenario's + own. The record the judge reads stays as written.""" + text = _straddling(at) + if where == "persona": + body = {"persona": {"name": text, "role": text}, "name": "s1"} + else: + body = {"persona": {"role": text}, "name": text} + body.update(instruction=text, tests="ok", use_case="Refund") + + persona, presented = _sent(body) + + for field in ("name", "role"): + assert persona[field] == _masked(text)[:255] + assert _FAKE_KEY[:6] not in json.dumps(persona) + assert presented["name"] == presented["role"] == text[:255] + + +@pytest.mark.parametrize("at", [222, 244]) +@pytest.mark.parametrize("where", ["use_case", "tests"]) +def test_a_key_straddling_the_cut_of_scenario_name_is_masked( + at: int, where: str +) -> None: + """cr2 M1: `display_scenario_name` cuts `use_case`, or `tests` when there is no `use_case` + and no `scenario_key` to make one from, at 255 characters.""" + text = _straddling(at) + body = {"persona": {"name": "Dana"}, "name": "s1", "instruction": "x", where: text} + + persona, presented = _sent(body, scenario_key="") + + assert persona["scenario_name"] == _masked(text)[:255] + assert _FAKE_KEY[:6] not in json.dumps(persona) + assert presented["scenario_name"] == text[:255] + + +def test_a_key_in_a_scenario_name_the_display_rewrites_is_masked() -> None: + """cr2 M1: with no `use_case`, `tests` or `scenario_key`, the shown name is the scenario's + `name` with `-` and `_` made spaces, which takes a key out of its shape (`ghp A1b2…`).""" + body = {"persona": {"name": "Dana"}, "name": f"call about {_FAKE_KEY} please"} + + persona, presented = _sent(body, scenario_key="") + + assert persona["scenario_name"] == "Call about [redacted] please" + assert _FAKE_KEY[4:20] not in json.dumps(persona) + assert presented["scenario_name"] == f"Call about ghp {_FAKE_KEY[4:]} please" + + +@pytest.mark.parametrize( + "body", + [ + {"persona": {"name": "N" * 300, "role": "R" * 300}, "name": "S"}, + {"persona": {"occupation": "O" * 300}, "name": "S" * 300}, + {"persona": {}, "name": ""}, + {"persona": "not a dict", "name": "S"}, + {"persona": {"name": "", "role": ""}, "name": "S"}, + {"name": "refund-for_bob", "tests": "", "use_case": ""}, + {}, + ], + ids=[ + "persona", + "occupation_and_fallback", + "caller", + "persona_not_a_dict", + "empty", + "name_rewritten", + "bare", + ], +) +def test_the_record_sent_is_the_record_shown_when_nothing_is_secret(body) -> None: + """cr2 M1, cr3 L4: cleaning first changes no honest field, so each fallback `persona_of` + takes is the same in the record sent.""" + for scenario_key in ("s1", ""): + persona, presented = _sent(body, scenario_key=scenario_key) + assert persona == {**presented, "scenario_key": scenario_key} + + +def test_a_key_the_shown_name_rewrites_into_shape_is_masked() -> None: + """cr3 M4: `call_AKIA…` isn't a key's shape (a key's own character comes before it), but the + shown name makes `_` a space, `Call AKIA…`. The clean after `persona_of` masks it.""" + body = {"persona": {"name": "Dana"}, "name": "call_AKIAFAKEFAKEFAKE1234 now"} + + persona, _ = _sent(body, scenario_key="") + + assert persona["scenario_name"] == "Call [redacted] now" + assert "AKIAFAKE" not in json.dumps(persona) + + +def test_loading_a_scenario_logs_nothing_and_the_send_logs_one_line( + caplog: pytest.LogCaptureFixture, +) -> None: + """cr3 L4: the clean and its log line are the send's. Loading a scenario cleans nothing, so + a run that loads scenarios and never sends them logs nothing.""" + body = {"persona": {"name": f"Dana {_FAKE_KEY}"}, "name": "s1"} + with caplog.at_level(logging.WARNING): + presented = ss._presented(body, scenario_key="s1") + read = ss._presented(body, scenario_key="s1", fields=True) + loaded = [r for r in caplog.records if r.name == "fi.alk.providers.agent"] + scenario = SimpleNamespace( + presented=presented, presented_fields=read, scenario_key="s1" + ) + ss._provision_payload("Run", [scenario], (), "", "chat") + sent = [r for r in caplog.records if r.name == "fi.alk.providers.agent"] + + assert loaded == [] + assert read["persona"]["name"] == f"Dana {_FAKE_KEY}" + assert [r.getMessage() for r in sent] == ["secrets redacted: kinds=github"] + + +def test_a_compiled_scenario_carries_the_fields_to_send() -> None: + """The one place a scenario is built from its document fills `presented_fields`.""" + import inspect + + assert ( + "presented_fields=_presented(body, scenario_key=scenario_key, fields=True)" + in inspect.getsource(ss._load_one) + ) diff --git a/tests/providers/test_providers_agent_build.py b/tests/providers/test_providers_agent_build.py index ba86233a6..fc69d8614 100644 --- a/tests/providers/test_providers_agent_build.py +++ b/tests/providers/test_providers_agent_build.py @@ -109,7 +109,7 @@ def test_a_refused_url_inside_prose_becomes_redacted_and_the_rest_stays( # One line for the call, however many URLs in it were refused (#271). [logged] = [record for record in caplog.records if record.name == "fi.alk.providers.agent"] - assert logged.getMessage() == "a URL was redacted whole: class=ValueError" + assert logged.getMessage() == "secrets redacted: url=ValueError" assert "hunter2" not in caplog.text @@ -140,11 +140,61 @@ def test_many_refused_urls_in_one_document_log_one_line( assert agent["behaviour"][0]["instructions"] == "[redacted]" assert agent["tools"][0]["parameters"]["properties"]["k"]["enum"] == ["[redacted]"] * 1001 [logged] = [record for record in caplog.records if record.name == "fi.alk.providers.agent"] - assert logged.getMessage() == "a URL was redacted whole: class=ValueError" + assert logged.getMessage() == "secrets redacted: url=ValueError" assert logged.exc_info is None assert "hunter2" not in caplog.text +def test_the_log_line_names_the_url_class_and_the_kinds_once_per_call( + caplog: pytest.LogCaptureFixture, +) -> None: + """TH-9500 (plan v1.1 section 2, S6): one line per call, naming what was removed, never + the text.""" + document = { + "a": "password=Zq8wYv3Lp9", + "b": ["https://u:p@h/x?k=1", "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8"], + "c": "Before https://admin:hunter2\uff03x@db.example/ after", + } + + with caplog.at_level(logging.DEBUG, logger="fi.alk.providers.agent"): + _redact(document) + + [logged] = [ + record for record in caplog.records if record.name == "fi.alk.providers.agent" + ] + assert logged.getMessage() == "secrets redacted: url=ValueError; kinds=github,named" + assert "Zq8wYv3Lp9" not in caplog.text and "hunter2" not in caplog.text + + +def test_the_log_line_names_the_kinds_alone_when_no_url_was_refused( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.DEBUG, logger="fi.alk.providers.agent"): + assert _redact("see https://bot:hun ter2@h/v1") == "see https://h/v1" + + assert [ + r.getMessage() for r in caplog.records if r.name == "fi.alk.providers.agent" + ] == ["secrets redacted: kinds=rule1"] + + +@pytest.mark.parametrize( + "text", + [ + "password: required", + "see https://h.example/x?k=1", + "password=[redacted]", + "no secret", + ], +) +def test_nothing_is_logged_when_no_secret_rule_changed_the_text( + caplog: pytest.LogCaptureFixture, text: str +) -> None: + with caplog.at_level(logging.DEBUG, logger="fi.alk.providers.agent"): + _redact(text) + + assert [r for r in caplog.records if r.name == "fi.alk.providers.agent"] == [] + + # -- TH-9491: a URL inside prose (plan v5.1) ------------------------------------------------------- # No shrinking: each assertion names its input, and a planted bug then fails in seconds. @@ -225,13 +275,104 @@ def _lone_urls(draw: Any) -> str: @_PROPERTY @given(_lone_urls()) def test_a_single_url_field_is_cleaned_as_the_base_cleans_it(url: str) -> None: - """B5: a single-URL field, after W1's join, gives the frozen base's result.""" + """B5: a single-URL field, after W1's join, gives the frozen base's result. TH-9500's rule 1 + first removes a userinfo whose text after "user:" a tab or a line break splits, which is + what the base's join used to read as one password. The oracle runs `leftovers`, so it can't + see a rule 1b condition that fires wrongly or not at all; the rows in + test_secret_shapes.py (R1B_CLEANED, R1B_KEPT, R1B_LIMITS) pin those (cr2 L4).""" from fi.alk.providers.agent import _START + from fi.alk.providers.secret_shapes import leftovers joined = url.replace("\t", "").replace("\r", "").replace("\n", "") if len(_START.findall(joined)) != 1: return - assert _redact(url) == _frozen_base_url(url), url + # A one-URL value runs step 1 on its joined token (v2.5.2, M-A), as chat does. + assert _redact(url) == _frozen_base_url(leftovers(joined)), url + assert _only_userinfo_and_query_values_taken(joined, leftovers(joined)), url + + +_QUERY_VALUE = re.compile(r"[^&#\s]*") + + +def _only_userinfo_and_query_values_taken(url: str, left: str) -> bool: + """What `leftovers` may take from a lone URL, checked without it (cr1 L1): the oracle above + runs `leftovers` itself, so a rule 1 bug moves both sides alike. Rule 1 takes the userinfo + right after the ://, up to and including its one @; rule 2 takes whole query values, each + after an =. Rule 1b takes the userinfo up to the first path segment's @. Nothing else + changes. It bounds what may go, not when: a 1b condition that fires wrongly or not at all + passes it, and the rows in test_secret_shapes.py pin those (cr2 L4).""" + head = url.index("://") + 3 + if left[:head] != url[:head]: + return False + rests = [url[head:]] + at = url.find("@", head) + if at >= 0: + rests.append(url[at + 1 :]) + slash = url.find("/", head) + path_at = url.find("@", slash) if slash >= 0 else -1 + if path_at >= 0: + rests.append(url[path_at + 1 :]) + return any(_values_taken(rest, left[head:]) for rest in rests) + + +def _values_taken(full: str, left: str) -> bool: + """Whether `left` is `full` less whole query values. A value taken is the whole run after + its =, so the next character left is one that ends a value, never the value's first.""" + at = kept = 0 + while at < len(full): + if at and full[at - 1] == "=": + value = _QUERY_VALUE.match(full, at).group() + if value and not left.startswith(value, kept): + at += len(value) + continue + if kept >= len(left) or full[at] != left[kept]: + return False + at += 1 + kept += 1 + return kept == len(left) + + +# R's adversarial lone URLs (change review r1, b5.py): hosts that are names, ports that read as +# values, userinfo split by breaks, rule 2's query on the next line, and brackets. +_ADVERSARIAL_URLS = [ + f"https://{userinfo}{host}{port}{path}" + for userinfo in ("", "u:p@") + for host in ("token", "secret", "password", "a") + for port in (":8443", ":0", ":{PORT}", "") + for path in ("/a1", "") +] + [ + "https://u:pa\nss@h/x", + "https://u:pa\tss@h/x", + "https://u:p[a\nb@h/x", + "https://u:a]b\n c@h/x", + "https://u:12\nab@h", + "https://u:{P}\nab@h", + "https://u:ab\r\ncd@h/x?y=1", + "https://u:ab\ncd@token:8443/a1", + "redis://:pw\nx@cache:6379/0", + "https://u:p@h/token=a1b2", + "https://h/x\n?a=1", + "https://h/x\n&a=1/b", + "https://u:p@h/x\n&a=1/b", + "https://u:x\ny@h/a\tb", + "https://u:p@[fd00::5]:8443/x", +] + + +def test_leftovers_takes_only_a_userinfo_and_query_values_from_a_lone_url() -> None: + """cr1 L1, beside B5 and without hypothesis: rule 1 and rule 2 checked independently.""" + from fi.alk.providers.secret_shapes import leftovers + + taken = {url: leftovers(url) for url in _ADVERSARIAL_URLS} + + assert { + url: left + for url, left in taken.items() + if not _only_userinfo_and_query_values_taken(url, left) + } == {} + # Each rule takes something here, so the check is not passing on untouched input. + assert taken["https://u:pa\nss@h/x"] == "https://h/x" + assert taken["https://h/x\n?a=1"] == "https://h/x\n?a=" @st.composite diff --git a/tests/providers/test_secret_shapes.py b/tests/providers/test_secret_shapes.py new file mode 100644 index 000000000..33b5ce114 --- /dev/null +++ b/tests/providers/test_secret_shapes.py @@ -0,0 +1,1510 @@ +"""TH-9500: secrets found by their shape, in the kit's one cleaner (plan v1.1, tests S1-S5b). + +Every key-shaped value here is fake. The corpus below is the design reviewers' case lists, +copied as data (W2), so a planted bug in a rule fails a named case. +""" + +from __future__ import annotations + +import itertools +import logging +import re +import subprocess +import sys + +import pytest + +from fi.alk.providers import secret_shapes +from fi.alk.providers.agent import ( + _LEADING_SCHEME, + _REDACTED, + _SPACE, + _URL_BREAKS, + _redact, + _redact_token, +) + + +def _base(text: str) -> str: + """TH-9491's URL step alone, frozen here: what the cleaner did before TH-9500.""" + if "://" not in text: + return text + if _LEADING_SCHEME.match(text): + joined = text.translate(_URL_BREAKS) + if not _SPACE.search(joined): + return _redact_token(joined, []) + return "".join( + part if not part or part.isspace() else _redact_token(part, []) + for part in _SPACE.split(text) + ) + + +def _logged(caplog: pytest.LogCaptureFixture) -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if record.name == "fi.alk.providers.agent" + ] + + +def test_the_marker_and_the_held_bound_match_their_owners() -> None: + from fi.alk.harness.hosted_chat_entrypoint import _HOLD_HARD + + assert secret_shapes.REDACTED == _REDACTED + assert secret_shapes._LONGEST_HELD_KEY == _HOLD_HARD + + +# -- S1: the drift test, against the kit's guardrails list ------------------------------------ + +_SAMPLES = { + "openai_api_key": "sk-" + + "FAKEfake0123456789ab" + + "T3BlbkFJ" + + "FAKEfake0123456789ab", + "openai_api_key_generic": "sk-proj-" + + "FAKEfake0123456789abcdefFAKEfake0123456789ab", + "aws_access_key": "AKIA" + "FAKEFAKE12345678", + "google_api_key": "AIza" + "FAKEfake0123456789FAKEfake012345678", + "github_pat": "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8", + "github_oauth": "gho_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8", + "github_user_token": "ghu_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8", + "github_server_token": "ghs_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8", + "github_refresh_token": "ghr_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8", + "github_fine_grained_pat": "github_pat_" + + "FAKEfake0123456789abcd" + + "_" + + "F" * 59, + "gitlab_pat": "glpat-" + "FAKEfake0123456789ab", + "slack_token": "xoxa-1234567890-1234567890-" + "FAKEfakeFAKEfakeFAKEfake", + "slack_bot_token": "xoxb-12345678901-12345678901-" + "FAKEfakeFAKEfakeFAKEfake", + "stripe_secret_key": "sk_live_" + "FAKEfake0123456789FAKEfa", + "sendgrid_api_key": "SG." + + "FAKEfake0123456789abcd" + + "." + + "FAKEfake0123456789" * 2 + + "abcdefg", + "anthropic_api_key": "sk-ant-" + "FAKEfake0123456789" * 2 + "abcd", + "huggingface_token": "hf_" + "FAKEfake0123456789FAKEfake01234567", + "npm_token": "npm_" + "FAKEfake0123456789FAKEfake0123456789", + "pypi_token": "pypi-AgEIcHlwaS5vcmc" + "FAKEfake0123456789" * 3, + "private_key_header": "-----BEGIN RSA PRIVATE KEY-----\nMIIEFAKEQUJDREVG\n" + "-----END RSA PRIVATE KEY-----", + "pgp_private_key": "-----BEGIN PGP PRIVATE KEY BLOCK-----\nlQOYBFAKEQUJDREVG\n" + "-----END PGP PRIVATE KEY BLOCK-----", +} +# Every other guardrails entry at confidence 0.9 or more, with why it isn't a prefix pattern. +_LEFT_OUT = { + "aws_secret_key": "(?i) start; kept as its own pattern, from its own regex (r4 M3)", + "azure_storage_key": "(?i) start; kept as its own pattern, from its own regex (r4 M3)", + "firebase_api_key": "(?i) start; a name covered by the named-value rule, value AIza", + "mailchimp_api_key": "no literal start; left out on purpose, it matches honest hex IDs", + "discord_token": "no literal start; the bare Discord bot token is a named limit", + "telegram_bot_token": "no literal start; the bare Telegram token is our own pattern", + "database_url": "(?i) start; TH-9491's URL step removes the userinfo", + "connection_string": "(?i) start, and its .* makes the regex quadratic", +} + + +def _guardrails() -> dict[str, tuple[str, float]]: + from fi.evals.guardrails.scanners import secrets + + return { + name: (pattern, confidence) + for listed in vars(secrets).values() + if isinstance(listed, list) and listed and isinstance(listed[0], tuple) + for pattern, name, confidence in listed + } + + +def _literal_prefix(pattern: str) -> str: + out = [] + at = 0 + while at < len(pattern): + char = pattern[at] + if char == "\\" and at + 1 < len(pattern) and pattern[at + 1] in ".-/": + out.append(pattern[at + 1]) + at += 2 + continue + if char in "\\[](){}?*+|^$.": + break + out.append(char) + at += 1 + return "".join(out) + + +def test_the_prefix_patterns_and_the_left_out_entries_are_the_guardrails_list() -> None: + entries = {name: entry for name, entry in _guardrails().items() if entry[1] >= 0.9} + prefixed = { + name + for name, (pattern, _) in entries.items() + if len(_literal_prefix(pattern)) >= 3 + } + + assert prefixed == set(_SAMPLES) + assert len(prefixed) == 21 + assert set(entries) - prefixed == set(_LEFT_OUT) + + +@pytest.mark.parametrize("name", sorted(_SAMPLES)) +def test_each_prefix_patterns_sample_is_masked(name: str) -> None: + pattern, _ = _guardrails()[name] + sample = _SAMPLES[name] + assert re.search(pattern, sample) + + out = _redact(f"Use this:\n{sample}\nthen stop.") + + assert sample.splitlines()[-2 if "\n" in sample else 0][-12:] not in out + assert out.startswith("Use this:\n") and out.endswith("\nthen stop.") + + +def _name_forms(words: list[list[str]]) -> list[str]: + """Every join a guardrails name allows: each optional word in or out, each join "", "_" + or "-", in lower and upper case and as PascalCase.""" + forms = set() + for kept in itertools.product( + *[[word] if required else [word, ""] for word, required in words] + ): + parts = [part for part in kept if part] + for joins in itertools.product(["", "_", "-"], repeat=len(parts) - 1): + name = parts[0] + "".join( + join + part for join, part in zip(joins, parts[1:]) + ) + forms |= {name, name.upper()} + forms.add( + parts[0].title() + + "".join(join + part.title() for join, part in zip(joins, parts[1:])) + ) + return sorted(forms) + + +_AWS = "wJalrXUtnFEMIFAKEK7MDENGbPxRfiCYFAKEKEY1" +_AZURE = ("FAKEazureFAKEkey0123456789abcdefFAKE+/" * 3)[:86] + "==" +_FIREBASE = "AIza" + "FAKEfake0123456789FAKEfake012345678" +_NAMED_ENTRIES = { + "aws_secret_key": ( + [["aws", True], ["secret", True], ["access", False], ["key", True]], + _AWS, + ), + "azure_storage_key": ( + [["azure", True], ["storage", False], ["account", False], ["key", True]], + _AZURE, + ), + "firebase_api_key": ( + [["firebase", True], ["api", False], ["key", True]], + _FIREBASE, + ), +} + + +@pytest.mark.parametrize("entry", sorted(_NAMED_ENTRIES)) +def test_every_name_form_a_named_entry_accepts_is_masked(entry: str) -> None: + """The drift test for the named-value entries (r4 L5, M3): every name form the entry's + regex accepts, through this module.""" + pattern, _ = _guardrails()[entry] + words, value = _NAMED_ENTRIES[entry] + leaked = [] + for form in _name_forms([[word, required] for word, required in words]): + for join in ("=", " = ", ": "): + text = f"{form}{join}{value}" + assert re.search(pattern, text), text + if value in _redact(text): + leaked.append(text) + assert leaked == [] + + +@pytest.mark.parametrize("join", ["=", ": "]) +def test_the_bare_telegram_token_is_masked(join: str) -> None: + token = "123456789:AA" + "FAKEfake0123456789FAKEfake0123456" + assert re.search(_guardrails()["telegram_bot_token"][0], token) + assert _redact(f"BOT{join}{token} ok") == f"BOT{join}{_REDACTED} ok" + + +# Round 1 (r1cases.py), with its fake key shapes. +GH = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8" +SKP = "sk-proj-" + "FAKEfake0123456789abcdefFAKEfake0123456789ab" +AK = "AKIA" + "FAKEFAKE12345678" +XB = "xoxb-12345678901-12345678901-" + "FAKEfakeFAKEfakeFAKEfake" +SLACK = "https://hooks.slack.com/services/" + "T00FAKE00/B00FAKE00/" + "FAKEfakeFAKEfakeFAKEfake" +TG = "https://api.telegram.org/bot123456789:AAFAKEfakeFAKEfakeFAKEfakeFAKEfake_x/sendMessage" +JWT = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJmYWtlIn0.FAKEsigFAKEsig" +PEM = "-----BEGIN RSA PRIVATE KEY-----\nMIIEFAKEFAKEFAKE\nABCDEFFAKE==\n-----END RSA PRIVATE KEY-----" + +R1_CASES = { + "page: ws in password": "https://bot:hun ter2@h/v1", + "page: lb in password": "see https://bot:hun\n ter2@h/v1 is the hook", + "page: tab in password": "see https://bot:hun\tter2@h/v1 is the hook", + "page: query next line": "see https://u:pw@h/x\n ?key=sk-1 and more", + "page: ?# in password": "https://bot:8812#x@db/", + "page: / in password": "https://bot:pa/ss@db", + "page: :// in password": "https://bot:pa://ss@db", + "bare ghp prose": f"Use {GH} to push.", + "bare sk-proj prose": f"key is {SKP}.", + "bare AKIA quoted": f'aws="{AK}"', + "bare xoxb json": f'{{"t": "{XB}"}}', + "slack hook": f"Post to {SLACK} daily", + "telegram": f"Call {TG} now", + "jwt prose": f"Bearer {JWT}", + "pem": f"key:\n{PEM}\nend", + "pem header only": "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAAFAKE\n(truncated)", + "assign pw": "password=hunter22", + "assign spaced": "token: s3cr3tvalue99", + "tab query": "see https://h/x\t?key=sk-1 more", + "mid-query break": "see https://h/x?a=1&\n key=sk-1 more", + "break before ? no indent": "see https://h/x\n?key=sk-1 more", + "nbsp query": "see https://h/x ?key=sk-1 more", + "nbsp password": "https://bot:hun ter2@h/v1", + # honest text + "H port+email": "Run at http://localhost:3000 and email support@acme.com", + "H query email": "https://host:8080/path?email=a@b.com", + "H npm scoped": "Install from https://registry.npmjs.org:443/@babel/core today", + "H medium": "Read https://medium.com/@user for more", + "H port then mention": "Open http://localhost:8080 and ping @oncall in Slack", + "H md amp": "See https://docs.x.com/a\n©=2024 Acme", + "H password required": "password: required", + "H token expires": "the token: expires hourly", + "H secret never": "secret: never tell anyone", + "H env": 'api_key=os.environ["OPENAI_API_KEY"]', + "H env placeholder": "Send the token: ${SLACK_BOT_TOKEN} in the header", + "H commit": "Fixed in 1ac329e16b0f2c9a8e7d6c5b4a3f2e1d0c9b8a7f", + "H uuid": "order 123e4567-e89b-12d3-a456-426614174000", + "H b64img": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", + "H order": "Order ID ORD-2024-000123 and SKU AKIAMODEL", + "H token count": "max_tokens: 4096 and token=streaming", +} + +# Round 1's p2 and p5 additions (round 3's cases.py). +A = "FAKEfake0123456789" +R1_EXTRA = { + "r1p2 clone x:ghp": f"https://x:{GH}@github.com/org/repo.git", + "r1p5 anthropic": "sk-ant-api03-" + (A * 6)[:95], + "r1p5 google": "AIza" + (A * 2)[:35], + "r1p5 gitlab": "glpat-" + (A * 2)[:20], + "r1p5 hf": "hf_" + (A * 2)[:34], + "r1p5 BOT_TOKEN": "BOT_TOKEN=Zq8wYv3Lp9Kd", + "r1p5 yaml client_secret": "client_secret: Zq8wYv3Lp9Kd2", + "r1p5 PEM PKCS8": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASC\n-----END PRIVATE KEY-----", +} + +# Round 2 (p1.py). +R2_CASES = { + "owl p0 mid-query 2": "https://h/x?a=1&\n key=sk-1", + "owl p0 slack": "https://hooks.slack.com/services/" + "T00000000/B00000000/" + "XXXXXXXXXXXXXXXXXXXXXXXX", + "owl p0 token sk-proj-abc": "token: sk-proj-abc", + "H site residue": "Visit https://site.com:8443?ref=me@x.com", + "H port/ email": "Login at http://localhost:8080/login as admin@acme.com", + "r1 digits ws": "https://bot:12 34@h/v1", + "r1 digits slash": "https://bot:8812/x@db/", + "r1 two ws runs": "https://bot:hun ter 2@h", + "r1 user dot": "https://my.bot:hun ter2@h", + "r1 user dash": "https://ci-bot:hun ter2@h", + "r1 empty user redis": "redis://:pa/ss@cache:6379/0", + "r1 empty user ws": "redis://:hun ter2@cache:6379/0", + "H a.com:x y@z": "see http://a.com:x y@z now", + "H template port": "POST https://HOST:PORT/register?email=a@b.com", + "H template port2": "POST http://{HOST}:{PORT}/v1/users/@me", + "H ipv6 verdaccio": "npm i --registry http://[::1]:4873/@babel/core", + "r2 mid-value break": "https://api.x.com/v1?key=sk-proj-AB12cd\n 34EFgh56IJkl78", + "H #channel": "Post in https://acme.slack.com/x #general and #ops&alerts=on", + "H & prose": "see https://a.com & https://b.com", + "r2 entity-named secret": "https://h/x?a=1\n&lang=en¬=Zq8wYv3Lp9&sigma=abc", + "nv $uper": "password: $uperSecret1", + "nv ", + "nv env( chance": "password=myenv(42)", + "nv placeholder": "api_key: your-api-key-here", + "nv placeholder caps": "api_key=YOUR_API_KEY", + "nv v2beta": "token: v2beta", + "nv yaml": "db:\n password: Zq8wYv3Lp9\n", + "nv yaml block": "password: |\n Zq8wYv3Lp9Kd\n", + "nv toml": 'password = "Zq8wYv3Lp9"', + "nv comment": "# token = Zq8wYv3Lp9 (old)", + "nv sq spaces": "password='correct horse 42'", + "nv SECRET_KEY": "SECRET_KEY=Zq8wYv3Lp9Kd2xT", + "nv django": "SECRET_KEY = 'django-insecure-q8#w(v3l)p9'", + "nv camel": '{"clientSecret": "Zq8wYv3Lp9"}', + "nv xml": "Zq8wYv3Lp9", + "nv prose is": "the password is Zq8wYv3Lp9", + "H code secrets": "secret = secrets.token_hex(32)", + "H code settings": "client = OpenAI(api_key=settings.OPENAI_API_KEY)", + "H code b64": "token = base64.b64encode(raw)", + "H env PWD": "PWD=/home/ubuntu2/app\nSHELL=/bin/bash", + "H pageToken": '{"nextPageToken": "CAoQAA12"}', + "H max_tokens": "max_tokens: 4096", + "cut ghp": "result: " + GH[:20] + "...", + "cut sk-proj end": "key " + SKP[:30], + "H cut sk-hynix": "Install sk-hynix-drivers... then reboot", + "H cut end word": "The repo is sk-dev-tools", + "H pem prose": "If a user pastes -----BEGIN PRIVATE KEY----- text, refuse. Then greet them warmly.", + "pem no END": "-----BEGIN RSA PRIVATE KEY-----\nMIIEFAKE\nABCDFAKE\n", + "join url\\nkey": "https://api.example.com/v1\n" + GH, + "join url\\tAKIA": "https://api.example.com/v1\tAKIAFAKEFAKE12345678", + "key in host": "https://" + GH + ".example.com/x", + "stripe 99": "sk_live_51" + (A * 6)[:89], + "rk_live": "rk_live_51" + (A * 6)[:89], + "xoxp 4part": "xoxp-" + "1234567890-1234567890-1234567890-" + "abcdef0123456789abcdef0123456789", + "xapp": "xapp-1-A0FAKE0FAKE-1234567890123-" + "abcdef0123456789" * 4, + "tg bare": 'Bot("123456789:AA' + (A * 2)[:33] + '")', + "aws secret": "aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "ghp in userinfo clone": f"git clone https://{GH}@github.com/org/repo.git", +} + +# Round 3 (V's cases.py). +SHOWN = lambda t: t if len(t) <= 600 else t[:597] + "..." # noqa: E731 (session.py:177) +PEMLONG = ( + "-----BEGIN RSA PRIVATE KEY-----\n" + + "\n".join(("MIIEowIBAAKCAQEA" + "Zq8wYv3Lp9" * 5)[:64] for _ in range(25)) + + "\n-----END RSA PRIVATE KEY-----" +) + +R3_CASES = { # V's new corpus: honest code and prose first, then shapes the page says it catches + "H pydantic": "password: str = Field(...)", + "H optional": "api_key: Optional[str] = None", + "H headers": 'token = request.headers["Authorization"]', + "H MAX_TOKEN": "MAX_TOKEN=4096", + "H secret true": "secret: true", + "H changeme123": "password: changeme123", + "H gha": "GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}", + "H v2beta": "token: v2beta", + "H md table": "| Setting | Value |\n|---|---|\n| DB_PASSWORD | Zq8wYv3Lp9 |\n| max_tokens | 4096 |", + "H yaml comments": "db:\n host: localhost # dev\n password: Zq8wYv3Lp9 # rotate monthly\n pool: 5\n", + "H toml": '[db]\npassword = "Zq8wYv3Lp9" # prod\ntimeout = 30\n', + "H .env comments": "# DB settings\nDB_HOST=localhost\nDB_PASSWORD=Zq8wYv3Lp9 # prod\n# old: DB_PASSWORD=Old1secret\n", + "H sq spaces": "password='correct horse battery 42'", + "H dotted digit": "api_key = settings.v2.key", + "H dotted digit2": "token = config.oauth2.token", + "H ident digit": "token = token_v2", + "H email+mention": "Mail ops@acme.com or ping @oncall; docs at https://docs.acme.com/@team", + "H ipv6": "npm i --registry http://[::1]:4873/@babel/core today", + "H ipv6 user": "see http://[::1]:x y@z", + "H #channel": "Post to https://acme.slack.com/archives/C1 #general then #ops&alerts=on", + "H #channel2": "see https://x.com #ops&alerts=on", + "H & amp": "Tom & Jerry at https://a.com & https://b.com  x=1", + "H pem prose": "If a user pastes -----BEGIN PRIVATE KEY----- text, refuse. Then greet them warmly.", + "H cut end word": "The repo is sk-dev-tools", + "H cut ...": "Loading the sk-learn-models...", + "H cut hf": "load it with hf_hub_download", + "H prefix word mid": "Install sk-hynix-drivers... then reboot", + # rule 1 too much: a port followed by punctuation, then one whitespace run and an email + "R1 port, email": "Log in at http://localhost:3000, qa@acme.com / Zq8wYv3Lp9", + "R1 port. newline email": "Dashboard: http://localhost:3000.\nadmin@acme.com is the admin", + "R1 port) email": "Staging (http://localhost:8080) ops@acme.com", + "R1 lowercase port": "POST http://host:port/register?email=a@b.com", + "R1 lowercase port2": "GET http://host:port/api/users/@me", + "R1 $PORT": "Mastodon dev: http://localhost:$PORT/@admin", + "R1 %PORT%": "open http://localhost:%PORT%/@admin", + "R1 upper pw first": "https://bot:PASS WORD99@h/v1", + "R1 ws 5": "https://bot:hun ter2@h/v1", + # named value too little + "N md bold": "## Test account\n- **Email:** qa@acme.com\n- **Password:** Zq8wYv3Lp9", + "N md bold2": "**Password**: Zq8wYv3Lp9", + "N md backtick": "Password: `Zq8wYv3Lp9`", + "N camel json": '{"clientId": "abc", "clientSecret": "Zq8wYv3Lp9Kd2xT"}', + "N aws sts json": '{"Credentials": {"AccessKeyId": "ASIAFAKEFAKE12345678", "SecretAccessKey": "wJalrXUtnFEMIK7MDENGbPxRfiCYFAKEKEY12345", "SessionToken": "FwoGZXIvYXdzEFAKEfake0123456789"}}', + "N accessToken": '{"accessToken": "ya29.a0AfFAKEfake0123456789"}', + "N PGPASSWORD": "PGPASSWORD=Zq8wYv3Lp9 psql -h db", + "N MYSQL_PWD": "MYSQL_PWD=Zq8wYv3Lp9", + "N DB_PASS": "DB_PASS=Zq8wYv3Lp9\nSMTP_PASS=Zq8wYv3Lp9", + "N php =>": "'password' => 'Zq8wYv3Lp9',", + "N go :=": 'password := "Zq8wYv3Lp9"', + "N cli flag": "docker login -u bot --password Zq8wYv3Lp9 registry.acme.com", + "N bearer": 'curl -H "Authorization: Bearer Zq8wYv3Lp9Kd2xTq7" https://api.acme.com', + "N yaml phrase": "password: correct horse battery staple", + "N url\\npassword": "https://api.example.com/v1\npassword=Zq8wYv3Lp9", + "N url\\nDB_PASSWORD": "https://api.example.com/v1\nDB_PASSWORD=Zq8wYv3Lp9", + # key shapes + "K discordapp": "Post to https://discordapp.com/api/webhooks/123456789012345678/" + + (A * 4)[:68], + "K discord": "Post to https://discord.com/api/webhooks/123456789012345678/" + + (A * 4)[:68], + "K akia host": "https://AKIAFAKEFAKE12345678.example.com/x", + "K pem crlf no END": "-----BEGIN PRIVATE KEY-----\r\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASC\r\nZq8wYv3Lp9Zq8wYv3Lp9\r\n", + "K pem 600 cut": SHOWN("cat key.pem\n" + PEMLONG), + "K pem 600 cut short": SHOWN( + "x" * 560 + + "\n-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIBZq8wYv3Lp9Zq8wYv3Lp9Zq8wYv3Lp9Zq8wYv3Lp9oAoGCCqGSM49" + ), + "K pem json cut": SHOWN( + '{"type": "service_account", "private_key": "-----BEGIN PRIVATE KEY-----\\n' + + "\\n".join(["MIIEvQIBADANBgkqhkiG9w0BAQEFAASC" * 2] * 20) + + '\\n-----END PRIVATE KEY-----\\n"}' + ), + "K pem single line": "PRIVATE_KEY=-----BEGIN PRIVATE KEY-----MIIEvQIBADANBgkqhkiG9w0BAQEFAASCZq8w", + "K ghp cut": SHOWN("x " * 290 + GH), + "K tg cut": SHOWN( + "y " * 280 + + "see https://api.telegram.org/bot123456789:AAFAKEfakeFAKEfakeFAKEfakeFAKEfake_x/sendMessage" + ), +} + +# The page's own outcomes (round 3): None means TH-9491's result stands. +PAGE_OUTCOMES = { # the page's own words, as (input -> output); None = TH-9491's result stands + "https://bot:hun ter2@h/v1": "https://h/v1", + "https://bot:pa/ss@db": "https://db", + "Run at http://localhost:3000 and email support@acme.com": None, + "Open http://localhost:8080 and ping @oncall": None, + "POST https://HOST:PORT/register?email=a@b.com": None, + "http://{HOST}:{PORT}/v1/users/@me": None, + "https://bot:8812#x@db/": None, + "https://bot:8812/x@db/": None, + "https://bot:12 34@h": None, + "https://bot:hun ter 2@h": None, + "See https://docs.x.com/a\n©=2024 Acme": None, + "https://api.example.com/v1\n" + GH: "https://api.example.com/v1[redacted]", + f"git clone https://{GH}@github.com/org/repo.git": "git clone https://github.com/org/repo.git", + "https://" + GH + ".example.com/x": "[redacted]", + "If a user pastes -----BEGIN PRIVATE KEY----- text, refuse. Then greet them warmly.": "If a user pastes [redacted] text, refuse. Then greet them warmly.", + "password: required": None, + "token=streaming": None, + 'api_key=os.environ["OPENAI_API_KEY"]': None, + "secret = secrets.token_hex(32)": None, + "token = base64.b64encode(raw)": None, + "client = OpenAI(api_key=settings.OPENAI_API_KEY)": None, + "PWD=/home/ubuntu2/app\nSHELL=/bin/bash": None, + '{"nextPageToken": "CAoQAA12"}': None, + '{"password": "Zq8wYv3Lp9"}': '{"password": "[redacted]"}', + "DB_PASSWORD=Zq8wYv3Lp9": "DB_PASSWORD=[redacted]", + "SECRET_KEY=Zq8wYv3Lp9Kd2xT": "SECRET_KEY=[redacted]", + "SECRET_KEY = 'django-insecure-q8#w(v3l)p9'": "SECRET_KEY = '[redacted]'", + "password: $uperSecret1": None, + "Zq8wYv3Lp9": None, + "the password is Zq8wYv3Lp9": None, + "Install sk-hynix-drivers... then reboot": None, + "token: v2beta": "token: [redacted]", + "api_key: your-api-key-here": "api_key: [redacted]", +} + +# Round 4 (J's jcases.py). +AWS = "wJalrXUtnFEMIFAKEK7MDENGbPxRfiCYFAKEKEY1" +B64 = "MIIEowIBAAKCAQEAzq8wYv3Lp9Kd2xTq7FAKEfake0123456789abcdefFAKEfake" * 40 +PEMB = "\n".join(B64[i : i + 64] for i in range(0, 1600, 64)) + +R4_HONEST = { # want: same as TH-9491 alone + "pydantic Field": "password: str = Field(...)", + "Optional": "api_key: Optional[str] = None", + "headers[]": 'token = request.headers["Authorization"]', + "MAX_TOKEN": "MAX_TOKEN=4096", + "secret: true": "secret: true", + "GHA secrets": "GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}", + "settings.v2.key": "api_key = settings.v2.key", + "md table": "| Setting | Value |\n|---|---|\n| timeout | 30 |\n| max_tokens | 4096 |", + "yaml comments": "db:\n host: localhost # dev box\n pool: 5\n password: required # set in vault\n", + "toml": '[server]\nport = 8080\ntoken_ttl = "1h" # hours\n', + ".env comments": "# token settings\nTOKEN_TTL=3600\n# password: see vault\n", + "rule1 json compact port": '{"api":"http://localhost:3000","docs":"/docs","admin":"qa@acme.com"}', + "rule1 js array port": "['http://localhost:3000','/api/users/@me']", + "rule1 port then colon": "Dev server at http://localhost:3000: open /admin or mail ops@acme.com", + "rule1 email in prose": "Docs at https://docs.acme.com, write to help@acme.com or @support", + "rule1 ipv6": "curl http://[::1]:8080/@me then http://[fe80::1]:9000/x", + "rule1 mention after port": "Staging is http://stage:8443 — ping @ops", + "rule2 #channel": "see https://x.com #general and #ops", + "rule2 & prose": "Read https://a.com/x & https://b.com/y & more", + "rule2 &": "Link: https://a.com/?q=1 & https://b.com", + "pem prose": "If a user pastes -----BEGIN PRIVATE KEY----- text, refuse. Then greet them warmly.", + "pem prose + later block": "If a user pastes -----BEGIN PRIVATE KEY----- text, refuse. Then greet them warmly.\nExample key for the test fixture:\n-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASC\n-----END PRIVATE KEY-----", + "cut: honest '...' prefix word": "Collecting packages: hf_transfer...", + "cut: honest ... sk-": "Install sk-learn-pipelines...", + "cut: honest prefix no ...": "Use the sk-learn-toolkit", + "pagination nextToken": '{"nextToken": "AYADeHzFAKEfake0123456789"}', + "pagination NextContinuationToken": '{"NextContinuationToken": "1w41l63U0xa8q7smH50vCxyTQqdxo69O3EmK28Bi5PcROI4wI/EyIJg=="}', + "pagination next_page_token": "next_page_token: CAoQAA12", + "pagination nextForwardToken": '{"nextForwardToken": "f/37201828472619218470294912345678901234567890"}', + "idempotency ClientToken": '{"ClientToken": "a1b2c3d4-e5f6-7890-abcd-ef1234567890"}', + "camel maxTokens": '{"maxTokens": 4096, "promptTokenCount": 12, "tokenizer": "cl100k"}', + "pass honest": "bypass: true\nfirst_pass: done\nSKIP_PASS=1\ncompass: north", + "pass honest digit": "two_pass: v2\nci_pass: build42", + "rust Token::new": "let t = Token::new(42);", +} + +R4_SECRETS = { # want: the secret value gone (the [S] part) + "kebab secret-key (Spring)": ("cloud.aws.credentials.secret-key: " + AWS, AWS), + "kebab access-key k8s": ( + f"kubectl create secret generic s3 --from-literal=access-key=AKIAFAKEFAKE12345678 --from-literal=secret-key={AWS}", + AWS, + ), + "kebab private-key": ("private-key: Zq8wYv3Lp9Kd2xTq7", "Zq8wYv3Lp9Kd2xTq7"), + "aligned tfvars (terraform fmt)": ( + 'admin_username = "admin"\npassword = "Zq8wYv3Lp9"', + "Zq8wYv3Lp9", + ), + "aligned yaml": ("user: admin\npassword: Zq8wYv3Lp9", "Zq8wYv3Lp9"), + "aligned ini": ( + "[db]\nhost = db.internal\npassword = Zq8wYv3Lp9\napi_key = Zq8wYv3Lp9Kd2xTq7", + "Zq8wYv3Lp9Kd2xTq7", + ), + "dotted password": ("DB_PASSWORD=Acme.Admin1", "Acme.Admin1"), + "dotted random": ("password=x7.Kq9vL2mPz", "x7.Kq9vL2mPz"), + "ya29 unquoted": ( + "GOOGLE_ACCESS_TOKEN=ya29.a0AfB_byCFAKEfake0123456789abcdefFAKE", + "ya29.a0AfB_byCFAKEfake0123456789abcdefFAKE", + ), + "ya29 quoted": ( + '{"access_token": "ya29.a0AfB_byCFAKEfake0123456789abcdefFAKE"}', + "ya29.a0AfB_byCFAKEfake0123456789abcdefFAKE", + ), + "yaml block scalar": ("password: |\n Zq8wYv3Lp9Kd\n", "Zq8wYv3Lp9Kd"), + "unquoted passphrase": ("password: correct horse battery 42", "battery 42"), + "rule1 trailing space before break": ( + "hook: https://bot:hun \n ter2@h/v1", + "ter2", + ), + "rule1 blank line": ("hook: https://bot:hun\n\n ter2@h/v1", "ter2"), + "pem END-less escaped JSON, no name": ( + SHOWN( + '{"key": "-----BEGIN RSA PRIVATE KEY-----\\n' + + PEMB.replace("\n", "\\n") + + '\\n-----END RSA PRIVATE KEY-----"}' + ), + B64[:40], + ), + "pem END-less flattened spaces": ( + SHOWN( + "TLS_KEY=-----BEGIN RSA PRIVATE KEY----- " + + PEMB.replace("\n", " ") + + " -----END RSA PRIVATE KEY-----" + ), + B64[:40], + ), + "pem END-less encrypted, cut": ( + SHOWN( + "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-128-CBC,3F2A9C1B7E4D6F80A1B2C3D4E5F60718\n\n" + + PEMB + + "\n-----END RSA PRIVATE KEY-----" + ), + B64[:40], + ), + "pem END-less Read tool + arrow": ( + SHOWN( + "\n".join( + f"{i + 1:6}→{line}" + for i, line in enumerate( + ("-----BEGIN RSA PRIVATE KEY-----\n" + PEMB).split("\n") + ) + ) + ), + B64[:40], + ), + "camel passwd/pwd": ( + '{"dbPwd": "Zq8wYv3Lp9", "rootPasswd": "Zq8wYv3Lp9"}', + "Zq8wYv3Lp9", + ), + "pass camel": ('{"smtpPass": "Zq8wYv3Lp9"}', "Zq8wYv3Lp9"), +} + +AZ = ("FAKEazureFAKEkey0123456789abcdefFAKE+/" * 3)[:86] + "==" +FB = "AIza" + "FAKEfake0123456789FAKEfake012345678" + +R4_DRIFT = [ + ("aws_secret_key", f"aws_secret_key = {AWS}"), + ("aws_secret_key", f"AWS_SECRET_ACCESS_KEY={AWS}"), + ("aws_secret_key", f"aws-secret-key: {AWS}"), + ("aws_secret_key", f"awssecretkey={AWS}"), + ("aws_secret_key", f"AWSSecretKey={AWS}"), + ("azure_storage_key", f"AZURE_STORAGE_KEY={AZ}"), + ("azure_storage_key", f"azure_key = {AZ}"), + ("azure_storage_key", f"azure-storage-key: {AZ}"), + ("azure_storage_key", f"AzureStorageAccountKey={AZ}"), + ("firebase_api_key", f"FIREBASE_API_KEY={FB}"), + ("firebase_api_key", f"firebasekey: {FB}"), + ("mailchimp_api_key", "MC key 0123456789abcdef0123456789abcdef" + "-us12"), + ( + "discord_token", + "MTAxNTI3OTM4NjQ5Nzc0MDAzMg" + ".GFAKEf." + "FAKEfake0123456789FAKEfake012", + ), + ("telegram_bot_token", "123456789:AAFAKEfakeFAKEfakeFAKEfakeFAKEfake_x"), + ("database_url", "postgres://app:Zq8wYv3Lp9@db.internal:5432/app"), + ("connection_string", "Server=db;Database=app;User Id=sa;Pwd=Zq8wYv3Lp9;"), + ("connection_string", "Host=db;Username=app;Password=Zq8wYv3Lp9;"), +] + + +# -- S2: the design page's cases, rule by rule -------------------------------------------------- + +_B64 = "MIIEowIBAAKCAQEAzq8wYv3Lp9Kd2xTq7FAKEfake0123456789abcdefFAKEfake" * 40 +_PEM_JSON = "\\n".join(_B64[i : i + 64] for i in range(0, 1600, 64)) +_SHOWN = lambda text: text if len(text) <= 600 else text[:597] + "..." # noqa: E731 +_GH = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8" +_PW = "Zq8wYv3Lp9" + +_MASKED = { + # Rule 1: a password broken by whitespace, /, ? or #, with up to 64 characters a side. + "rule1 space": ("https://bot:hun ter2@h/v1", "https://h/v1"), + "rule1 break + indent": ( + "see https://bot:hun\n ter2@h/v1 is the hook", + "see https://h/v1 is the hook", + ), + "rule1 slash": ("https://bot:pa/ss@db", "https://db"), + "rule1 empty user": ("redis://:hun ter2@cache:6379/0", "redis://cache:6379/0"), + "rule1 64 a side": ("https://bot:" + "a" * 64 + " b@h/v1", "https://h/v1"), + # Rule 2: a query after a break. + "rule2 next line": ( + "see https://u:pw@h/x\n ?key=sk-1 and more", + "see https://h/x\n ?key= and more", + ), + "rule2 query ends in &": ( + "see https://h/x?a=1&\n key=sk-1 more", + "see https://h/x\n key= more", + ), + # A named value: up to 16 spaces or tabs of alignment, or up to 4 whitespace with a break. + "named json": ('{"password": "Zq8wYv3Lp9"}', '{"password": "[redacted]"}'), + "named env": ("DB_PASSWORD=Zq8wYv3Lp9", "DB_PASSWORD=[redacted]"), + "named django": ( + "SECRET_KEY = 'django-insecure-q8#w(v3l)p9'", + "SECRET_KEY = '[redacted]'", + ), + "named 16 spaces": ( + "password" + " " * 16 + "= " + _PW, + "password" + " " * 16 + "= [redacted]", + ), + "named break after :": ("password:\n " + _PW, "password:\n [redacted]"), + "named space break after :": ("password: \n " + _PW, "password: \n [redacted]"), + "named break after =": ("password=\n " + _PW, "password=\n [redacted]"), + "named break before :": ("password\n : " + _PW, "password\n : [redacted]"), + "named break before =": ("password\n = " + _PW, "password\n = [redacted]"), + "named md bold label": ("- **Password:** " + _PW, "- **Password:** [redacted]"), + "named md bold name": ("**Password**: " + _PW, "**Password**: [redacted]"), + "named kebab": ( + "cloud.aws.credentials.secret-key: " + _PW, + "cloud.aws.credentials.secret-key: [redacted]", + ), + "named camel": ( + '{"clientSecret": "Zq8wYv3Lp9Kd2xT"}', + '{"clientSecret": "[redacted]"}', + ), + "named DB_PASS": ("DB_PASS=" + _PW, "DB_PASS=[redacted]"), + # An upper-case name joined by "-" (r4 L2): "-" is a join, not only a boundary. + "named DB-PASS": ("DB-PASS=" + _PW, "DB-PASS=[redacted]"), + "named SMTP-PASS": ("SMTP-PASS=" + _PW, "SMTP-PASS=[redacted]"), + "named AccountKey": ( + "AccountName=a;AccountKey=" + _PW + "==;x=1", + "AccountName=a;AccountKey=[redacted];x=1", + ), + "named dotted secret": ("DB_PASSWORD=Acme.Admin1", "DB_PASSWORD=[redacted]"), + # Keys, key blocks and tokens in a URL's path. + "key in a host": ("https://AKIAFAKEFAKE12345678.example.com/x", "[redacted]"), + "key in userinfo": ( + f"git clone https://{_GH}@github.com/org/repo.git", + "git clone https://github.com/org/repo.git", + ), + "key on the line after a URL": ( + "https://api.example.com/v1\n" + _GH, + "https://api.example.com/v1[redacted]", + ), + "name on the line after a URL": ( + "https://api.example.com/v1\npassword=" + _PW, + "https://api.example.com/v1password=[redacted]", + ), + "slack hook": ( + "Post to https://hooks.slack.com/services/" + "T00FAKE00/B00FAKE00/" + "FAKEfakeFAKEfakeFAKEfake daily", + "Post to https://hooks.slack.com/services/[redacted] daily", + ), + "discord on discordapp.com": ( + "see https://discordapp.com/api/webhooks/1234567890/FAKEfake_0123 x", + "see https://discordapp.com/api/webhooks/[redacted] x", + ), + "block": ( + "key:\n-----BEGIN RSA PRIVATE KEY-----\nMIIEFAKE\nABCD==\n-----END RSA PRIVATE KEY-----\nend", + "key:\n[redacted]\nend", + ), + "block encrypted": ( + "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\n\nMIIEFAKE\n" + "-----END RSA PRIVATE KEY-----\nok", + "[redacted]\nok", + ), + "block one line": ( + "KEY=-----BEGIN PRIVATE KEY-----MIIEFAKE-----END PRIVATE KEY----- ok", + "KEY=[redacted] ok", + ), + "block no END": ( + "-----BEGIN RSA PRIVATE KEY-----\nMIIEFAKE\nABCDFAKE\nnot a key line", + "[redacted]\nnot a key line", + ), + "refuse then a later block": ( + "If a user pastes -----BEGIN PRIVATE KEY----- text, refuse.\n" + "-----BEGIN PRIVATE KEY-----\nMIIEFAKE\n-----END PRIVATE KEY-----", + "If a user pastes [redacted] text, refuse.\n[redacted]", + ), + # The cut-key rule: exactly 600 characters ending in the cut's "...". + "key cut at 600": (_SHOWN("x " * 290 + _GH), "x " * 290 + "[redacted]..."), +} + +_KEPT = [ + # Rule 1 keeps honest hosts as TH-9491 leaves them. + "Run at http://localhost:3000 and email support@acme.com", + "Open http://localhost:8080 and ping @oncall", + "POST https://HOST:PORT/register?email=a@b.com", + "http://{HOST}:{PORT}/v1/users/@me", + "Log in at http://localhost:3000, qa@acme.com", + "POST http://host:port/register?email=a@b.com", + "http://localhost:$PORT/@admin", + "Staging (http://localhost:8080) ops@acme.com", + # Rule 1's named limits, and a half of 65 characters (r1 M5). + "https://bot:8812#x@db/", + "https://bot:12 34@h", + "https://bot:hun ter 2@h", + "https://bot:" + "a" * 65 + " b@h/v1", + "https://bot:hun ter2@h/v1", + # Rule 1 needs text on both sides of the run: a mention right after it isn't a password. + "https://bot:abc @oncall", + "see https://bot:hunter2 @team now", + # Rule 2 keeps an HTML entity. + "See https://docs.x.com/a\n©=2024 Acme", + # Named values: references, code and honest values. + "password: required", + "token=streaming", + 'api_key=os.environ["OPENAI_API_KEY"]', + "secret = secrets.token_hex(32)", + "token = base64.b64encode(raw)", + "client = OpenAI(api_key=settings.OPENAI_API_KEY)", + "api_key = settings.v2.key", + "PWD=/home/ubuntu2/app\nSHELL=/bin/bash", + '{"nextPageToken": "CAoQAA12"}', + "two_pass: v2", + "let t = Token::new(42);", + "password: $uperSecret1", + "Zq8wYv3Lp9", + "the password is Zq8wYv3Lp9", + "password" + " " * 17 + "= " + _PW, + # A quoted value for each reference form (r1 M5). + 'token: "{{ secrets.TOKEN_1 }}"', + 'token: "$TOKEN_1"', + 'token: "${TOKEN_1}"', + 'token: ""', + 'token: "os.environ.get(TOKEN_1)"', + 'token: "process.env.TOKEN_1"', + 'token: "env(TOKEN_1)"', + # A break with 5 whitespace characters on either side of the = or : (r1 M2). + "password:\n " + _PW, + "password=\n " + _PW, + "password\n : " + _PW, + "password\n = " + _PW, + # The cut-key rule needs both the length and the "..." (r4 L1). + "Collecting packages: hf_transfer...", + "Install sk-hynix-drivers... then reboot", + "The repo is sk-dev-tools", + # 601 characters, so not the cut. + "x " * 289 + _GH[:20] + "...", +] + + +@pytest.mark.parametrize("name", sorted(_MASKED)) +def test_the_pages_masked_cases(name: str) -> None: + text, expected = _MASKED[name] + assert _redact(text) == expected + + +# Written out, not read from the module: a name dropped there must fail here. +_PAGINATION_NAMES = ( + "ClientToken", + "NextContinuationToken", + "NextToken", + "continuationToken", + "nextPageToken", + "nextToken", + "pageToken", +) + + +@pytest.mark.parametrize("name", _PAGINATION_NAMES) +def test_every_pagination_name_keeps_its_value(name: str) -> None: + """Each "not a name" is kept, not only the ones the page quotes.""" + text = '{"%s": "CAoQAA12"}' % name + assert _redact(text) == text + + +@pytest.mark.parametrize("text", _KEPT, ids=range(len(_KEPT))) +def test_the_pages_kept_cases_come_out_as_th9491_leaves_them(text: str) -> None: + assert _redact(text) == _base(text) + + +# cr1 M1 (change review r1, R's hostname.py): a host whose name ends in a name, with a port, +# as Compose and Kubernetes service names do. Its ":" opens no value, so the URL stays as +# TH-9491 leaves it. +_HOST_NAMES = [ + "Call the issuer at http://auth-token:8080/v1/issue before each run.", + "Vault runs at http://vault-secret:8200/v1/kv in compose.", + "The token service: http://token:9000/oauth/token", + "GET http://api-key:3000/keys then POST", + "Use http://secret:8080 for local dev.", + "Use http://my-password:5000/reset to reset.", + "postgres://app@db-password:5432/app", + "see http://Auth-Token:8080/v1 now", + "base_url: http://token-svc:8080/v1", + "http://auth-token.internal:8080/v1", +] + + +@pytest.mark.parametrize("text", _HOST_NAMES, ids=range(len(_HOST_NAMES))) +def test_a_host_named_like_a_secret_keeps_its_port_and_path(text: str) -> None: + assert _redact(text) == _base(text) + + +def test_a_named_value_after_a_url_in_one_run_is_still_masked() -> None: + """cr1 M1: only a port after the name keeps it; this "=" opens a value.""" + out = _redact("url=https://api.x.com,token=Zq8wYv3Lp9Kd next words") + + assert "zq8wyv3lp9kd" not in out.lower() + assert out.endswith(" next words") + + +# -- Rule 1b: an @ before the password's / (design addendum v2.5.2) ----------------------------- +# Each cleaned row as (input, output). The chat stream runs these same rows at every split +# (test_builder_chat.py), and outbound runs the first four (test_outbound.py). +R1B_CLEANED = { + # Dileep's #278 threads P1 and P3: a password holding / or #, rule 1's own. + "Dileep P1": ( + "postgres://admin:Xk9/Qp+Zr8@db.example.com:5432/app", + "postgres://db.example.com:5432/app", + ), + "Dileep P3": ("https://user:pa#ss@host.example/x", "https://host.example/x"), + # Ash's two: an @ in the password, and an @ in the user. + "Ash P@ss": ( + "postgres://admin:P@ss/Xk9@db.example.com:5432/app", + "postgres://db.example.com:5432/app", + ), + "Ash smtp": ( + "smtp://me@gmail.com:app/pass@smtp.gmail.com:587", + "smtp://smtp.gmail.com:587", + ), + "redis": ("redis://:p@w/x9@cache.internal:6379/0", "redis://cache.internal:6379/0"), + ":99": ( + "postgres://admin:P@ss:99/Xk9@db.example.com/app", + "postgres://db.example.com/app", + ), + # M2's three hosts. + "localhost": ( + "postgres://admin:P@ss/Xk9@localhost/app", + "postgres://localhost/app", + ), + "a single label": ("postgres://admin:P@ss/Xk9@db/app", "postgres://db/app"), + "IPv6 with a port": ( + "postgres://admin:P@ss/Xk9@[::1]:5432/app", + "postgres://[::1]:5432/app", + ), + # M3 and M-A: a one-URL value runs step 1 on its joined token. + "M3 broken at Xk\\n9": ( + "postgres://admin:P@ss/Xk\n9@db.example.com:5432/app", + "postgres://db.example.com:5432/app", + ), + "M-A broken at Pa\\nss": ( + "postgres://admin:Pa\nss@w/x9@db.example.com/app", + "postgres://db.example.com/app", + ), + "M-A redis": ( + "redis://:pa\nss@w/x9@cache.internal:6379/0", + "redis://cache.internal:6379/0", + ), + # M-B's hosts. + "M-B postgres_db:5432": ( + "postgres://admin:P@ss/Xk9@postgres_db:5432/app", + "postgres://postgres_db:5432/app", + ), + "M-B my_db/app": ("postgres://admin:P@ss/Xk9@my_db/app", "postgres://my_db/app"), + "M-B a backtick": ("`redis://:p@w/x9@redis`", "`redis://redis`"), + "M-B db:${DB_PORT}": ( + "postgres://admin:P@ss/Xk9@db:${DB_PORT}/app", + "postgres://db/app", + ), + "M-B db:$DB_PORT": ( + "postgres://admin:P@ss/Xk9@db:$DB_PORT/app", + "postgres://db/app", + ), + "M-B db?sslmode": ("postgres://admin:P@ss/Xk9@db?sslmode=disable", "postgres://db"), + "M-B redis#0": ("redis://:p@w/x9@redis#0", "redis://redis"), + "M-B )": ("(amqp://u:p@w/x9@rabbitmq)", "(amqp://rabbitmq)"), + "M-B ,": ("amqp://u:p@w/x9@rabbitmq, then", "amqp://rabbitmq, then"), + "M-B ;": ("amqp://u:p@w/x9@rabbitmq; then", "amqp://rabbitmq; then"), + "M-B >": ("", ""), + # TH-9491 refuses the stray ], as it would with no password. + "M-B ]": ("[amqp://u:p@w/x9@rabbitmq]", "[[redacted]"), + "M-B zoned IPv6": ( + "postgres://admin:P@ss/Xk9@[fe80::1%25en0]:5432/app", + "postgres://[fe80::1%25en0]:5432/app", + ), + # M4: one sorted, merged list of rule 1's and 1b's spans. + "M4 1b then rule 1": ( + "see postgres://admin:P@ss/Xk9@db.example.com/app and https://bot:hun ter2@h/v1 now", + "see postgres://db.example.com/app and https://h/v1 now", + ), + "M4 rule 1 then 1b": ( + "see https://bot:hun ter2@h/v1 and postgres://admin:P@ss/Xk9@db.example.com/app now", + "see https://h/v1 and postgres://db.example.com/app now", + ), + "M4 overlapping": ("a://u:x://v:p@h/x@host.example", "a://host.example"), +} +# M1 and M-C: a user with no password keeps its host, and a path's @ before no host is left +# alone, as TH-9491 leaves them. +R1B_KEPT = [ + "http://admin@localhost:8080/bob@example.com", + "ssh://git@gitlab.example.com:2222/ops@infra.example/repo.git", + "https://ci@registry.example.com:8443/lodash@4.17.21", + "ssh://git@host.example:22/org@team/repo", + "Read https://medium.com/@user for more", + "Install from https://registry.npmjs.org:443/@babel/core today", + "http://admin@[::1]/bob@example.com", + "ssh://git@[fd00::5]/org@team/repo", + "http://admin@[::1]:8080/bob@example.com", + # A user with no : at all. + "https://ci@registry.example.com/lodash@4.17.21", + "ssh://git@github.com/org@team/repo", + # A path's @ with no host after it: rule 1b doesn't hold, and TH-9491 removes the userinfo. + "Mail https://u:p@h.example/to@~team now", +] +# The addendum's named limits, pinned as they come out. +R1B_LIMITS = { + # L1: the host moves; the password is still removed. + "https://u:p@api.example.com/a@b.com": "https://b.com", + "https://u:p@api.example.com/lodash@4.17.21": "https://4.17.21", + "https://u:p@api.example.com/react@18?dev": "https://18", + "git+https://u:${TOKEN}@git.example.com/repo.git@main": "git+https://main", + # L2: the scp form inside a scheme. + "git+ssh://git@github.com:org/repo@v1.2.0": "git+ssh://v1.2.0", + "git+ssh://git@github.com:org/repo@main": "git+ssh://main", + # M-C: a port rule 1 doesn't read. + "http://admin@localhost:{{port}}/bob@example.com": "http://example.com", + "http://admin@localhost:$(PORT)/bob@example.com": "http://example.com", + "http://admin@localhost:/bob@example.com": "http://example.com", + # Still leaks, as on the base: a password that reads as a port, an empty segment, two /, + # a segment over its bound, %40, and a line break inside the span in prose. + "smtp://me@gmail.com:1234/pass@smtp.gmail.com": "smtp://gmail.com:1234/pass@smtp.gmail.com", + "postgres://admin:P@ss/@db.example.com/app": "postgres://ss/@db.example.com/app", + "postgres://admin:P@ss/a/b@db.example.com/app": "postgres://ss/a/b@db.example.com/app", + "postgres://admin:P@ss/" + "a" * 65 + "@db.example.com/app": "postgres://ss/" + + "a" * 65 + + "@db.example.com/app", + "postgres://admin:P%40ss/Xk9@db.example.com/app": "postgres://admin/Xk9@db.example.com/app", + "see postgres://admin:P@ss/Xk\n9@db.example.com/app now": "see postgres://ss/Xk\n9@db.example.com/app now", +} + + +@pytest.mark.parametrize("name", list(R1B_CLEANED)) +def test_rule_1b_cleans_the_addendums_rows(name: str) -> None: + text, expected = R1B_CLEANED[name] + assert _redact(text) == expected + assert _redact(expected) == expected + + +@pytest.mark.parametrize("text", R1B_KEPT, ids=range(len(R1B_KEPT))) +def test_rule_1b_leaves_these_to_th9491(text: str) -> None: + assert _redact(text) == _base(text) + + +@pytest.mark.parametrize("text", list(R1B_LIMITS), ids=range(len(R1B_LIMITS))) +def test_rule_1bs_named_limits_are_pinned(text: str) -> None: + assert _redact(text) == R1B_LIMITS[text] + + +def test_rule_1bs_bounds_hold_at_each_edge() -> None: + """The bounds (L3): 64 before the first : or @, 64 from the last @ to the /, 1-64 from the + / to the path's @, and 129 up to the last @.""" + tail = "@db.example.com/app" + assert ( + _redact("postgres://" + "a" * 64 + ":P@ss/Xk9" + tail) + == "postgres://db.example.com/app" + ) + assert "Xk9" in _redact("postgres://" + "a" * 65 + ":P@ss/Xk9" + tail) + assert ( + _redact("postgres://admin:P@" + "s" * 64 + "/Xk9" + tail) + == "postgres://db.example.com/app" + ) + assert "Xk9" in _redact("postgres://admin:P@" + "s" * 65 + "/Xk9" + tail) + assert ( + _redact("postgres://admin:P@ss/" + "x" * 64 + tail) + == "postgres://db.example.com/app" + ) + # Before the authority's last @: 129 characters cleans, 130 doesn't. + for before, cleaned in ((129, True), (130, False)): + user = "a:P@" + "q" * (before - 4) + "@" + out = _redact("postgres://" + user + "s/Xk9" + tail) + assert (out == "postgres://db.example.com/app") is cleaned, before + + +def test_rule1_returns_one_sorted_list_with_overlaps_merged() -> None: + """M4: `remove` needs spans in order and apart.""" + text = "a://u:x://v:p@h/x@host.example" + assert secret_shapes.rule1(text) == [(4, 18)] + assert secret_shapes.remove(text, secret_shapes.rule1(text)) == "a://host.example" + + +# -- Change review r2 (cr2) ---------------------------------------------------------------------- + + +def test_a_one_url_value_cut_at_600_masks_its_cut_key() -> None: + """cr2 M5: the join-first branch keeps the raw value's 600-character test, so a key cut short + at the end is masked to it, joined or not.""" + for between in ("/tokens/", "\n/tokens/"): + value = _SHOWN("https://h.example/" + "a" * 547 + between + _GH) + assert len(value) == 600 + + out = _redact(value) + + assert out.endswith("/tokens/[redacted]..."), between + assert "ghp_A1b2" not in out + + +@pytest.mark.parametrize( + "text, expected", + [ + ("password:2024/Zq8wYv3Lp9 ok", "password:[redacted] ok"), + ("see token:8080/x9 now", "see token:[redacted] now"), + ], +) +def test_a_name_with_a_port_like_value_outside_a_host_is_masked( + text: str, expected: str +) -> None: + """cr2 M5: only a name in a URL's host is skipped for its port (cr1 M1); outside a host, a + value that begins with digits and a / is still a value.""" + assert _redact(text) == expected + + +def test_a_one_url_value_whose_password_held_a_space_is_joined_whole() -> None: + """cr2 L1, a named limit: once rule 1 removes a password broken by a space, the value is one + URL, and TH-9491's join takes the next line into it. No secret goes; the line is lost.""" + assert _redact("redis://:pa ss@h:6379\nREDIS_DB=0") == "redis://h" + + +@pytest.mark.parametrize( + "text, expected", + [ + # `_URL_END` stops at the next ://, so the URL after a span in a host is kept. + ( + "see x://password=Zq8w1,b/https://h.example/docs now", + "see [redacted]https://h.example/docs now", + ), + # Rule 1b's host: localhost with a trailing dot, and with a port rule 1 doesn't read. + ( + "postgres://admin:P@ss/Xk9@localhost.:5432/app", + "postgres://localhost.:5432/app", + ), + ( + "postgres://admin:P@ss/Xk9@localhost:{{port}}/app", + "postgres://localhost/app", + ), + ], +) +def test_the_url_end_and_rule_1bs_localhost_host(text: str, expected: str) -> None: + """cr2 L5.""" + assert _redact(text) == expected + + +def test_a_url_query_before_a_key_body_doesnt_defeat_the_cut_rules() -> None: + """r1 M1: Terraform state cut at 600 with a query URL before private_key_pem. Both + 600-character tests are made on the raw value, which step 2 would shorten.""" + state = _SHOWN( + '{"attributes": {"endpoint": "https://sts.amazonaws.com/?Action=AssumeRole&Version=2011-' + '06-15", "private_key_pem": "-----BEGIN RSA PRIVATE KEY-----\\n' + + _PEM_JSON + + '\\n-----END RSA PRIVATE KEY-----\\n"}}' + ) + assert len(state) == 600 + + out = _redact(state) + + assert "MIIEowIBAAKC" not in out and "FAKEfake0123" not in out + assert out.endswith('"private_key_pem": "[redacted]') + # A key cut short at 600 is masked to the end, behind a query too. + shown = _SHOWN("see https://h.example/x?a=1 then " + "y " * 270 + _GH) + assert _redact(shown).endswith(" [redacted]...") + + +def test_the_page_outcomes_hold() -> None: + """The page's own outcomes, as round 3 listed them: None means TH-9491's result stands.""" + for text, expected in PAGE_OUTCOMES.items(): + assert _redact(text) == (_base(text) if expected is None else expected), text + + +# -- S3 and S4: the corpus ---------------------------------------------------------------------- + + +def _corpus() -> dict[str, str]: + cases = {} + for label, group in ( + ("r1", R1_CASES), + ("r1 extra", R1_EXTRA), + ("r2", R2_CASES), + ("r3", R3_CASES), + ("r4 honest", R4_HONEST), + ): + cases.update({f"{label}: {name}": text for name, text in group.items()}) + cases.update({f"r4 secret: {name}": text for name, (text, _) in R4_SECRETS.items()}) + cases.update( + {f"r4 drift {i}: {name}": text for i, (name, text) in enumerate(R4_DRIFT)} + ) + return cases + + +CORPUS = _corpus() +# Pieces of the fake secrets in the corpus. None may survive cleaning, outside the limits below. +SECRET_PARTS = ( + "A1b2C3d4E5f6G7h8I9j0", + "FAKEfake0123456789", + "FAKEFAKE12345678", + "FAKEfakeFAKEfakeFAKEfake", + "AAFAKEfake", + "FAKEsigFAKEsig", + "MIIEFAKE", + "ABCDEFFAKE", + "ABCDFAKE", + "MHcCA", + "hunter2", + "ter2", + "Zq8wYv3Lp9", + "s3cr3tvalue99", + "wJalrXUtnFEMI", + "MIIEowIBAAKC", + "MIIEvQIBADAN", + "b3BlbnNzaC1rZXkt", + "Acme.Admin1", + "x7.Kq9vL2mPz", + "ya29.a0Af", + "FAKEazureFAKEkey", + "battery", + "correct horse", + "sk-1", + "pa/ss", + "pa://ss", + "q8#w(v3l)p9", + "FwoGZXIv", + "XXXXXXXXXXXXXXXX", + "abcdef0123456789abcdef", + "A0FAKE0FAKE", + "8812", + "ghp_A1b2C3d4", +) +# Cases where a secret is kept, each a limit the design page names. +SECRETS_KEPT = { + "r1: page: ?# in password": "a password that starts with digits", + "r2: r1 digits slash": "a password that starts with digits", + "r2: nv ": "=> isn't matched (r3 H3)", + "r3: N go :=": ":= isn't matched (r3 H3)", + "r3: N cli flag": "--password X isn't matched (r3 H3)", + "r3: N bearer": "Authorization: Bearer X isn't matched (r3 H3)", + "r3: N yaml phrase": "an unquoted passphrase with spaces (r4 L6)", + "r4 secret: yaml block scalar": "a YAML block scalar (r4 L6)", + "r4 secret: unquoted passphrase": "an unquoted passphrase with spaces (r4 L6)", + "r4 secret: rule1 blank line": "a blank line inside a password (r4 L3)", + "r4 secret: camel passwd/pwd": "dbPwd isn't a name (r4 L6)", + "r4 secret: pass camel": "smtpPass, a best-effort form (TH-9555)", + "r4 drift 15: connection_string": "Pwd= isn't a name", + "r4 drift 11: mailchimp_api_key": "Mailchimp's shape is left out on purpose", + "r4 drift 12: discord_token": "the bare Discord bot token is a named limit (r4 L5)", + "r2: cut ghp": "a key cut short in a text that isn't the 600-character cut (r4 L1)", + "r2: cut sk-proj end": "shorter than the key's least length, and not the cut (r4 L1)", + "r4 honest: pagination nextToken": "a pagination name's value is honest (r4 L2)", +} +# Text with no secret that the rules change, each as the design page says they would. +HONEST_CHANGED = { + "r2: H a.com:x y@z": "rule 1 reads 'x y' as a password broken by a space", + "r2: nv placeholder": "a placeholder 16 or more characters long (page, what can go wrong)", + "r2: nv v2beta": "token: v2beta is masked (page, what can go wrong)", + "r2: H pem prose": "the bare header is masked; the instruction after it is kept", + "r3: H changeme123": "a letter and a digit", + "r3: H v2beta": "token: v2beta is masked (page, what can go wrong)", + "r3: H ident digit": "an identifier with a digit isn't code", + "r3: H #channel2": "a markdown anchor after an address loses its values (r3 L6)", + "r3: H pem prose": "the bare header is masked; the instruction after it is kept", + "r4 honest: pem prose": "the bare header is masked; the instruction after it is kept", + "r4 honest: pagination next_page_token": "a best-effort form (TH-9555)", + "r4 honest: pagination nextForwardToken": "a best-effort form (TH-9555)", +} + + +def test_every_secret_in_the_corpus_is_masked() -> None: + leaked = { + name: [part for part in SECRET_PARTS if part in text and part in _redact(text)] + for name, text in CORPUS.items() + if name not in SECRETS_KEPT + } + assert {name: parts for name, parts in leaked.items() if parts} == {} + + +def test_honest_text_in_the_corpus_comes_out_unchanged() -> None: + """IDs, hashes, base64, code, markdown and pagination names: as TH-9491 leaves them.""" + changed = [ + name + for name, text in CORPUS.items() + if not any(part in text for part in SECRET_PARTS) + and name not in HONEST_CHANGED + and _redact(text) != _base(text) + ] + assert changed == [] + for name in HONEST_CHANGED: + assert _redact(CORPUS[name]) != _base(CORPUS[name]), name + + +def test_cleaning_the_corpus_twice_changes_nothing_and_logs_once( + caplog: pytest.LogCaptureFixture, +) -> None: + """S4: idempotence, and one log line per call, none on the second pass.""" + for name, text in CORPUS.items(): + caplog.clear() + with caplog.at_level(logging.DEBUG, logger="fi.alk.providers.agent"): + once = _redact(text) + first = _logged(caplog) + caplog.clear() + twice = _redact(once) + second = _logged(caplog) + assert twice == once, name + assert len(first) <= 1 and second == [], name + + +# -- S5 and S5b: speed -------------------------------------------------------------------------- + +_SETUP = r""" +import logging, random, string, time +logging.disable(logging.WARNING) +from fi.alk.providers.agent import _redact +GH = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8" +def rep(unit): + return (unit * (M // len(unit) + 1))[:M] +def dense(): + pick, unreserved, out, size = random.Random(9), string.ascii_letters + string.digits + "-._~", [], 0 + while size < M: + host = "".join(pick.choice(string.ascii_lowercase) for _ in range(8)) + key = "".join(pick.choice(unreserved) for _ in range(10)) + sentence = f"Call https://u{size}:p{key}@{host}.example/v{size % 97}?key={key} then reply. " + out.append(sentence) + size += len(sentence) + return "".join(out) +""" + +# The budget for each 5 MiB shape below, best of 3 (design "Speed", and TH-9553's four shapes). +# Set by the Coordinator for 3.12.3 under load (change review r1, cr1 M2). +_S5_BUDGET_S = 1.5 +_MIB = 1024 * 1024 +_SHAPES = { + "prose with no key": 'rep("The quick brown fox jumps over the lazy dog. ")', + "prose with names and no key": 'rep("Each token costs; the secret is batching, the password policy is long. ")', + "TH-9491 address-dense": "dense()", + "eyJ repeated": 'rep("eyJ")', + "digits then :AA": 'rep("1234567:AA")', + "an unterminated key block": '"-----BEGIN RSA PRIVATE KEY-----\\n" + rep("A" * 64 + "\\n")', + "-----BEGIN headers with no END": 'rep("-----BEGIN RSA PRIVATE KEY-----")', + "overlapping named values": 'rep("password=")', + "TH-9553 URLs with a key in each host": 'rep("https://" + GH + ".example.com/x ")', + "TH-9553 a one-URL value with a name on every line": '"https://h.example/x" + rep("\\npassword=Zq8w1")', + "TH-9553 token: abc repeated": 'rep("token: abc ")', +} + + +# cr1 H1 (change review r1): a name in a URL's host, repeated with no whitespace, and one such +# span before a long run of scheme characters. Each took over 10 s at 1 MiB or less when +# _url_around searched forward from every span. 1 MiB, best of 3, under 1 s: pinned for +# linearity, not timed at 5 MiB (the Coordinator). +_H1_BUDGET_S = 1.0 +_H1_SHAPES = { + "a name in a host, repeated": 'rep("a://password=Zq8w1,b/")', + "one host span, then a run, then /y://z": '"a://password=Zq8w1,b/" + "x" * (M - 27) + "/y://z"', +} + + +def _best_of_three(shape: str, expression: str, size: int) -> float: + code = ( + _SETUP + + f"M = {size}\n" + + f"value = {expression}\n" + + "best = 9.0\n" + + "for _ in range(3):\n" + + " start = time.perf_counter()\n" + + " _redact(value)\n" + + " best = min(best, time.perf_counter() - start)\n" + + "print(best)\n" + ) + try: + done = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + timeout=60, + check=True, + ) + except subprocess.TimeoutExpired: + pytest.fail(f"{shape}: three cleans took more than 60 s") + return float(done.stdout.strip()) + + +@pytest.mark.parametrize("shape", list(_SHAPES)) +def test_five_mib_cleans_within_the_budget(shape: str) -> None: + best = _best_of_three(shape, _SHAPES[shape], 5 * _MIB) + assert best < _S5_BUDGET_S, f"{shape}: {best:.3f} s" + + +@pytest.mark.parametrize("shape", list(_H1_SHAPES)) +def test_a_name_in_a_url_host_cleans_in_linear_time(shape: str) -> None: + best = _best_of_three(shape, _H1_SHAPES[shape], _MIB) + assert best < _H1_BUDGET_S, f"{shape}: {best:.3f} s" + + +# Rule 1b's shapes and rule 1's, 5 MiB each (design addendum v2.5.2, "Speed"; the Coordinator's +# call): the cleaner with 1b against the same build without it, `secret_shapes.rule1b` swapped +# out in the same process, best of 5 each, interleaved. With 1b, at most 2 times without it, +# and under 5 s. A shape with no @ runs no 1b match, so it can't fail on the ratio: the 5 s +# ceiling is what pins it, as the Coordinator's call has it (cr2 L6). +_RULE1B_RATIO = 2.0 +_RULE1B_CEILING_S = 5.0 +_RULE1_SHAPES = { + "rule 1's scheme://u: with no @": 'rep("https://u:")', + # One @ at the very end: rule 1 must find it once, not again from every scheme://. + "rule 1's scheme://u: then one @": 'rep("https://u:")[:-1] + "@"', + "TH-9553 https://u:p repeated": 'rep("https://u:p ")', + "rule 1's u:p@h/x@, no host": 'rep("https://u:p@h/x@ ")', + "rule 1's u:p@h/x@h.io": 'rep("https://u:p@h/x@h.io ")', + # Owl's reviewer S's and Hawk's reviewer T's shapes for 1b. + "1b a://u:": 'rep("a://u:")', + "1b a://u:p@h/x@": 'rep("a://u:p@h/x@")', + "1b a://:@/@": 'rep("a://:@/@")', + "1b a://u@": 'rep("a://u@")', + "1b a:// then x: 40 times": 'rep("a://" + "x:" * 40)', + "1b a:// then x@ 40 times": 'rep("a://" + "x@" * 40)', + "1b each part at its bound, no host": 'rep("a://" + "u" * 30 + ":" + "p" * 29 + "@"' + ' + "q" * 60 + "/" + "s" * 60 + "@ ")', + "1b port guard, :1 95 times": 'rep("a://u@" + ":1" * 95 + "/")', + "1b port guard, :{ 95 times": 'rep("a://u@" + ":{" * 95 + "/")', + "1b port guard, u@h:80/x@h.io": 'rep("a://u@h:80/x@h.io ")', + "1b a label run after the path's @": 'rep("a://u:p@h/x@" + "a" * 60 + "!")', + "1b a label host, one URL": 'rep("a://u:p@h/x@db/")', + "1b a dotted run after the path's @": 'rep("a://u:p@h/x@" + "a." * 30 + "!")', + "1b a hit, one URL": 'rep("a://u:p@h/x@h.io")', + "1b a hit and rule 1 together": 'rep("a://u:x://v:p@h/x@h.io ")', +} + + +def _with_and_without_1b(shape: str, expression: str) -> tuple[float, float]: + """The best of 5 without rule 1b and with it, interleaved in one process.""" + code = ( + _SETUP + + f"M = {5 * _MIB}\n" + + f"value = {expression}\n" + + "from fi.alk.providers import secret_shapes\n" + + "with_1b = secret_shapes.rule1b\n" + + "best = {False: 99.0, True: 99.0}\n" + + "for _ in range(5):\n" + + " for on in (False, True):\n" + + " secret_shapes.rule1b = with_1b if on else (lambda text: [])\n" + + " start = time.perf_counter()\n" + + " _redact(value)\n" + + " best[on] = min(best[on], time.perf_counter() - start)\n" + + "print(best[False], best[True])\n" + ) + try: + done = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + timeout=150, + check=True, + ) + except subprocess.TimeoutExpired: + pytest.fail(f"{shape}: ten cleans took more than 150 s") + without, with_1b = map(float, done.stdout.split()) + return without, with_1b + + +@pytest.mark.parametrize("shape", list(_RULE1_SHAPES)) +def test_rule_1b_costs_at_most_twice_the_cleaner_without_it(shape: str) -> None: + without, with_1b = _with_and_without_1b(shape, _RULE1_SHAPES[shape]) + assert with_1b <= _RULE1B_RATIO * without, ( + f"{shape}: {with_1b:.3f} s, {without:.3f} s without" + ) + assert with_1b < _RULE1B_CEILING_S, f"{shape}: {with_1b:.3f} s" + + +# Rule 1's three S5 shapes, 5 MiB against 1 MiB (the Coordinator's call, cr3): a slowdown that +# grows faster than the text fails here, at 5 times the size, without waiting for a timeout. +_RULE1_GROWTH = 6.0 +_RULE1_LINEAR_SHAPES = ( + "rule 1's scheme://u: with no @", + "rule 1's scheme://u: then one @", + "TH-9553 https://u:p repeated", +) + + +@pytest.mark.parametrize("shape", _RULE1_LINEAR_SHAPES) +def test_rule_1s_shapes_grow_with_the_text(shape: str) -> None: + """Best of 5 at each size, interleaved in one process: 5 MiB at most 6 times 1 MiB.""" + code = ( + _SETUP + + "sizes = (1024 * 1024, 5 * 1024 * 1024)\n" + + "values = {}\n" + + "for M in sizes:\n" + + f" values[M] = {_RULE1_SHAPES[shape]}\n" + + "best = dict.fromkeys(sizes, 99.0)\n" + + "for _ in range(5):\n" + + " for size in sizes:\n" + + " start = time.perf_counter()\n" + + " _redact(values[size])\n" + + " best[size] = min(best[size], time.perf_counter() - start)\n" + + "print(best[sizes[0]], best[sizes[1]])\n" + ) + try: + done = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + timeout=150, + check=True, + ) + except subprocess.TimeoutExpired: + pytest.fail(f"{shape}: ten cleans took more than 150 s") + one, five = map(float, done.stdout.split()) + assert five <= _RULE1_GROWTH * one, ( + f"{shape}: {five:.3f} s at 5 MiB, {one:.3f} s at 1" + ) + + +class _Spy: + """Stands in for a compiled pattern and counts every call made on it.""" + + calls = 0 + + def __init__(self, pattern: re.Pattern[str]) -> None: + self._pattern = pattern + + def __getattr__(self, name: str): + method = getattr(self._pattern, name) + if not callable(method): + return method + + def counted(*args, **kwargs): + _Spy.calls += 1 + return method(*args, **kwargs) + + return counted + + +def _spied(value): + if isinstance(value, re.Pattern): + return _Spy(value) + if isinstance(value, tuple): + return tuple(_spied(item) for item in value) + if isinstance(value, list): + return [_spied(item) for item in value] + if isinstance(value, dict): + return {key: _spied(item) for key, item in value.items()} + return value + + +def test_text_with_no_literal_and_no_name_calls_no_pattern( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """S5b (r1 M5): the str.find pre-check, not the regex engine's own literal search, is what + passes such text over.""" + for name, value in list(vars(secret_shapes).items()): + spied = _spied(value) + if spied is not value and not callable(value): + monkeypatch.setattr(secret_shapes, name, spied) + _Spy.calls = 0 + + assert ( + secret_shapes.find("The quick brown fox jumps over the lazy dog. " * 2000) == [] + ) + assert _Spy.calls == 0 + # The spies do see a call when there is something to look at. + assert secret_shapes.find("password=" + _PW) + assert _Spy.calls > 0