From a9a80ee29ababc2737ed8fa56951303cf9af6da7 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Thu, 12 Nov 2026 10:00:46 +0530 Subject: [PATCH 1/2] fix(harness): catch secrets by shape in the kit's one cleaner, in chat, outbound and the scenario send TH-9491 cleans a URL's userinfo, query and fragment. This adds what it can't see, in three steps of agent._redact (secret_shapes.py): 1. on the raw value: rule 1 (a password broken by whitespace, ?, # or /), rule 1b (a password or user holding @ before the password's /) and rule 2 (a query after a break); a value that is one URL joins first; 2. TH-9491's URL step, unchanged; 3. key patterns (GitHub, OpenAI, Anthropic, AWS, Slack, Stripe, Google and others), JWTs, PEM key blocks, Telegram bot URLs, and named values with guards for honest text. The chat stream's seam (ChatHold) never cuts inside a match, so every split streams what cleaning the whole reply stores; before prose, past N, it cuts only at a line break and holds while a URL's userinfo or its own :// is still open. Outbound runs the full cleaner. The scenario send builds its persona record from cleaned fields; the session's activity line is cleaned before its cut. The reviewers' offset sweeps are tests: every case streams equal to the stored message, with no part of its secret in any event. Fake provider keys in tests are split literals. Refs: TH-9500 Signed-off-by: khushalsonawat --- src/fi/alk/harness/hosted_chat_entrypoint.py | 226 ++- src/fi/alk/harness/outbound.py | 46 +- src/fi/alk/harness/scenario_source.py | 52 +- src/fi/alk/harness/session.py | 3 +- src/fi/alk/providers/agent.py | 117 +- src/fi/alk/providers/secret_shapes.py | 1359 +++++++++++++++ tests/harness/test_builder_chat.py | 1362 +++++++++++++++ tests/harness/test_contract_writer.py | 2 +- tests/harness/test_hosted_entrypoint.py | 20 + tests/harness/test_outbound.py | 87 +- tests/harness/test_reading_entry.py | 2 +- .../test_scenario_source_provision_agent.py | 209 ++- tests/providers/test_providers_agent_build.py | 149 +- tests/providers/test_secret_shapes.py | 1510 +++++++++++++++++ 14 files changed, 5042 insertions(+), 102 deletions(-) create mode 100644 src/fi/alk/providers/secret_shapes.py create mode 100644 tests/providers/test_secret_shapes.py diff --git a/src/fi/alk/harness/hosted_chat_entrypoint.py b/src/fi/alk/harness/hosted_chat_entrypoint.py index c1a300d22..41f8cc23d 100644 --- a/src/fi/alk/harness/hosted_chat_entrypoint.py +++ b/src/fi/alk/harness/hosted_chat_entrypoint.py @@ -39,6 +39,7 @@ ) from .tools import schema from .session import ARTIFACT, DONE, RESULT, STAGE_IDLE_TIMEOUT_SECONDS, TEXT, TOOL, Event, Stage +from ..providers.secret_shapes import ChatHold from ..providers.agent import ( _LETTER, _QUERY_OR_FRAGMENT, @@ -167,6 +168,11 @@ # cleaned token by token, split at every whitespace; any whitespace is then a cut, and each # piece is cleaned the prose way too, so it splits at the same tokens and keeps its whitespace. # +# The secret shapes cross whitespace, though: a password broken by a space or a line break, a +# query on the next piece, a named value and its value, and a key block. So the cut is narrowed by +# ChatHold (providers/secret_shapes.py): any whitespace is a cut only where none of those is +# still open or has just closed, and the shed below asks it where it may split (TH-9500). +# # The holdback is bounded. Past _HOLD_SOFT characters with no cut, what is held is shed: # - with a ? or # after its first URL start, the rest of the token is that URL's query. The held # text is sent cleaned (the URL with its host and path), and the rest of the token is dropped @@ -191,6 +197,13 @@ class _DeltaCleaner: """The cleaned text of one streamed message, let out piece by piece.""" + # A run of the breaks the one-URL join removes (cr3 H1, `_line_cut`). + _BREAK_RUN = re.compile(r"[\t\r\n]+") + # Where a URL's authority ends, and so its userinfo (`_line_cut`). + _AUTHORITY_END = re.compile(r"[/?#]") + # A scheme's characters, any of them first (`_learn_past`, cr3 M6). + _RUN_CHARS = re.compile(r"[A-Za-z0-9+.\-]*") + def __init__(self) -> None: self.held = "" self.dropping = False @@ -202,6 +215,14 @@ def __init__(self) -> None: # Before prose is known: a space has come after the opening began, with only whitespace # after it so far. self._spaced = False + # Past the soft bound, while the opening's run may still end in something other than + # "://" (`_learn_past`). + self._past: str | None = None + # Before prose: the joined index of a cut `_line_cut` held at 0, while the held text has + # only grown since (cr4 M2). + self._held_cut: int | None = None + # Where a cut may fall without splitting a secret's match (TH-9500). + self._hold = ChatHold() def feed(self, text: str) -> str: out: list[str] = [] @@ -223,19 +244,21 @@ def feed(self, text: str) -> str: return "".join(out) text = text[cut.start() :] self.dropping = False - last = _ANY_CUT.search(text[::-1]) if self.prose else None - if last is None: - self.held += text - else: - end = len(text) - last.start() - out.append(self._clean(self.held + text[:end])) - self.held = text[end:] + # TH-9500: the seam cuts where this would, at the last whitespace once the text is + # prose, but never into a match that is still open or has just closed: rule 1, rule 2, a + # named value and a key block all cross whitespace. + self._hold.prose = self.prose + end = self._hold.cut(self.held, text) + whole = self.held + text + if end: + out.append(self._clean(whole[:end])) + self.held = whole[end:] if len(self.held) > _HOLD_SOFT: out.append(self._shed()) return "".join(out) def flush(self) -> str: - held, self.held, self.dropping = self.held, "", False + held, self.held, self.dropping, self._held_cut = self.held, "", False, None return self._clean(held) if held else "" def _learn(self, text: str) -> None: @@ -252,13 +275,38 @@ def _learn(self, text: str) -> None: return self._spaced = True if len(self._opening) >= _HOLD_SOFT: + self._learn_past(text) return - self._opening += text[: _HOLD_SOFT - len(self._opening)] + taken = _HOLD_SOFT - len(self._opening) + self._opening += text[:taken] run = _SCHEME_RUN.match(self._opening) rest = self._opening[run.end() :] if run else self._opening # Not a letter first, or a scheme's run followed by anything but "://". if run is None or (rest and not "://".startswith(rest[:3])): self.prose = True + elif len(self._opening) >= _HOLD_SOFT: + # The opening's run, or its "://", isn't over at the soft bound: what follows still + # decides, as cleaning the whole text decides it (cr3 M6). Held as one URL, a run + # that ends in a line break would have the URL lines after it glued onto it, and + # their queries read as its path. + self._past = rest + self._learn_past(text[taken:]) + + def _learn_past(self, text: str) -> None: + """Past the soft bound: the opening's run, then "://" or not (`_past` holds what + followed the run so far; "" while the run goes on, None once decided).""" + past = self._past + if past is None or not text: + return + if not past: + end = self._RUN_CHARS.match(text).end() + if end == len(text): + return + text = text[end:] + past = (past + text)[:3] + if not "://".startswith(past): + self.prose = True + self._past = None if self.prose or past == "://" else past def _clean(self, piece: str) -> str: if not self.prose: @@ -266,37 +314,157 @@ def _clean(self, piece: str) -> str: # is after the message's last character (no part of the stripped reply), so it is # kept as it came. body = piece.rstrip() - return _redact(body.translate(_URL_BREAKS)) + piece[len(body) :] + return self._joined_clean(body) + piece[len(body) :] # The space in front only keeps the cleaner from reading the piece as one URL, which # would drop its tabs and line breaks; the prose way keeps them, and the space comes off. return _redact(" " + piece)[1:] + def _line_cut(self, raw: str, joined: str, at: int) -> int: + """Before prose, where in the held text a cut the branches put at `at` in its joined form + falls: at the start of the break run before the line that holds it (cr3 H1). 0 when + nothing may go yet. + + The branches find URL starts on the joined text, where a line is glued onto the next + line's scheme (`DB_PASSWORD=V` then `https://…` reads `Vhttps://`). Cut there, the name + would go out alone and its value be cleaned as a scheme, lower-cased. Cleaning the whole + text masks line starts before the join, so a cut keeps lines whole, and each piece sent is + masked and joined as the whole is. + + Nothing is sent while 0 holds, so the held text only grows; the answer for the same `at` + stays 0 until a later URL start moves `at`. It is kept (`_held_cut`), so a long hold + isn't walked again on every delta (cr4 M2).""" + if at == self._held_cut: + return 0 + # The break runs, found once: the walk goes back over them by index, so a line costs + # its own length however far the held text reaches back. + runs = [(found.start(), found.end()) for found in self._BREAK_RUN.finditer(raw)] + end, line_at, i = len(raw), len(joined), len(runs) + while True: + # Back over the lines from the end, to the one that holds `at`. + i -= 1 + if i < 0: + self._held_cut = at + return 0 + run, brk = runs[i] + line_at -= end - brk + if at >= line_at: + break + end = run + # A name that begins the line before, with its value on this line (`token:` then `V`), is + # one match: the two lines stay together. Such a value is a span found with the line + # before and not without it. + if i: + from ..providers.secret_shapes import line_starts + + before = runs[i - 1][1] + stop = runs[i + 1][0] if i + 1 < len(runs) else len(raw) + own = {span[0] + brk - 1 for span in line_starts(raw[brk - 1 : stop])} + with_before = line_starts(raw[before - 1 : stop]) + if any( + span[0] + before - 1 >= brk and span[0] + before - 1 not in own + for span in with_before + ): + line_at -= run - before + run = runs[i - 1][0] + # A URL before the cut keeps its userinfo with its host: the text after the cut, up to + # the next ://, may still be that userinfo (`postgres://admin:Xk9/Qp+Zr`, then `8@db…`, + # a rule 1b password holding a /). So while an @ comes before that :// (the last URL's + # own, which the cut is at or before), nothing goes. Nor does it while the text before + # the cut ends in a userinfo still open: an @ after a : since the last ://, with no /, + # ? or # between that @ and the cut (`…:P@ss/Xk9@`, then the host). The last :// is + # looked for in the joined text, where a scheme broken by a line break is whole again. + scheme = joined.rfind("://", 0, line_at) + # A cut at the start of a line that is only a URL's own `://` (its scheme on the line + # before) would send the scheme alone and leave the userinfo to be cleaned without it, + # so nothing goes. A known limit: an honest reply with such a line and then more than + # about 12 K of glued URLs reaches the hard bound and streams `[redacted]`. + inside = joined.rfind("://", 0, line_at + 3) + if inside >= 0 and line_at - 3 < inside <= line_at: + self._held_cut = at + return 0 + if scheme >= 0: + following = joined.find("://", line_at) + sign = joined.rfind("@", scheme, line_at) + if "@" in joined[line_at:following] or ( + sign > joined.find(":", scheme + 3, sign) >= 0 + and not self._AUTHORITY_END.search(joined, sign, line_at) + ): + self._held_cut = at + return 0 + return run + + def _joined_clean(self, text: str) -> str: + """Held text before prose, cleaned: masked at its line starts, then joined, as the whole + text is (cr2 H1). The kinds masked go on this clean's one log line.""" + from ..providers.agent import _line_starts_joined + + found: list[str] = [] + joined = _line_starts_joined(text, found) + return _redact(joined, hits=found) + def _shed(self) -> str: - held = self.held + # 1. The limit: feed calls this once more than _HOLD_SOFT characters are held. + # 2. The split, from the seam (TH-9500): None when nothing it holds is open, so the + # branches below run on all of the held text; otherwise the start of the held text's + # first open match that may not be cut: a key block, or a key, whose hold starts at + # its URL piece when it is in one. + split = self._hold.shed_cut(self.held) + # 6. A limit of 0: the held text opens with such a match, so nothing goes out yet. + if split == 0: + return "" + rest = "" + if split is not None: + self.held, rest = self.held[:split], self.held[split:] + # 3. The branches, run on the text before the split. Before prose they run on the + # joined text, and a cut keeps lines whole (`_line_cut`, cr3 H1). + held = head = raw = self.held if not self.prose: if self._spaced: - self.held = "" + self.held, self._held_cut = "", None return self._clean(held) - self.held = held = held.translate(_URL_BREAKS) + held = raw.translate(_URL_BREAKS) starts = [] - for found in _START.finditer(held): - letter = _LETTER.search(held, found.start(), found.end() - 3) - starts.append(letter.start() if letter else found.end() - 3) + for start in _START.finditer(held): + letter = _LETTER.search(held, start.start(), start.end() - 3) + starts.append(letter.start() if letter else start.end() - 3) + + def cut(at: int) -> tuple[str, str]: + # The held text before `at` (an index into `held`), cleaned, and the rest, held again. + if self.prose: + return _redact(held[:at]), held[at:] + if len(raw) != len(held): + at = self._line_cut(raw, held, at) + if not at: + return "", raw + # Text goes, so the held text's indexes move: a held cut no longer stands. + self._held_cut = None + return self._joined_clean(raw[:at]), raw[at:] + if not starts: tail = _SCHEME_TAIL.search(held) - keep = tail.start() if tail else len(held) - self.held = held[keep:] - return _redact(held[:keep]) - if _QUERY_OR_FRAGMENT.search(held, starts[0]): - self.held, self.dropping = "", True - return _redact(held) - out = "" - if starts[-1] > 0: - out = _redact(held[: starts[-1]]) - self.held = held = held[starts[-1] :] - if len(held) > _HOLD_HARD: - self.held, self.dropping = "", True - out += _REDACTED + out, self.held = cut(tail.start() if tail else len(held)) + elif _QUERY_OR_FRAGMENT.search(held, starts[0]): + out = _redact(held) if self.prose else self._joined_clean(raw) + self.held, self.dropping, self._held_cut = "", True, None + else: + out, self.held = "", raw + if starts[-1] > 0: + out, self.held = cut(starts[-1]) + # The held text itself is counted, breaks too: the join drops them, but they are held, + # and the walk over them (`_line_cut`) is bounded by this. + if len(self.held) > _HOLD_HARD: + self.held, self.dropping, self._held_cut = "", True, None + out += _REDACTED + # 5. The drop rule. A key inside a held URL piece is never a cut point: the split above + # is that piece's start or nothing. A split is a piece's start, so the token a branch + # dropped ends before it, and no fragment of it is kept. When the text the branches + # ran on ends in whitespace its last token is complete, so nothing is left to drop + # and the next honest word is kept (r1 L6). Before prose a tab or a line break is no + # token's end: the one-URL join removes it. + if self.prose and self.dropping and head[-1:].isspace(): + self.dropping = False + # 4. The tail from the split is held again. + self.held += rest return out diff --git a/src/fi/alk/harness/outbound.py b/src/fi/alk/harness/outbound.py index bac09d384..0a08438c4 100644 --- a/src/fi/alk/harness/outbound.py +++ b/src/fi/alk/harness/outbound.py @@ -70,6 +70,7 @@ ) from fi.simulate.results.futureagi import MAX_TARGET_LATENCY_TURNS +from ..providers.agent import _log_redacted, _redact_text from .job import FailureDomain, HarnessStage logger = logging.getLogger(__name__) @@ -277,28 +278,37 @@ def _mask_userinfo(match: re.Match[str]) -> str: def redact_outbound_text(value: str, extra_secret_values: tuple[str, ...] = ()) -> str: """Scrubs a single free-text field before it can leave the sandbox on any of the three channels (outbound-channels.md v1.3 "Redaction (enforced before emit)"; hosted-execution- - seams.md v1.11 §3 "any outbound projection ... redacts userinfo"). Two things, applied in - order: - - 1. URL userinfo (`_USERINFO_PATTERN`/`_mask_userinfo`, above) -- a password is masked and the - username kept (matching the contract's own `postgresql://harness:***@...` example); a bare - token/username-only userinfo (no `:`) is masked outright, since that shape is how a bearer - token appears, not a username. - 2. `extra_secret_values` -- exact-substring replacement for a caller-supplied list of secret - values. Always `()` today: the adapter (P10) is what will know the job's declared secrets - and pass them in -- this parameter exists now so `build_event_record`/`build_result_receipt` - never need to change shape when that wiring lands. - - NOT a general secret-content scanner -- the contract's "same secret-content scan as the - artifact sealer" is a separate, sealer-side obligation. This is the narrow subset this module - can enforce on every string field it controls without false-positiving on ordinary diagnostic - text. + seams.md v1.11 §3 "any outbound projection ... redacts userinfo"). Three things, applied in + order (TH-9500): + + 1. `extra_secret_values` -- exact-substring replacement for a caller-supplied list of secret + values. First, so a declared secret is matched whole before the cleaner below can mask + part of it and leave the rest unmatched. Always `()` today: the adapter (P10) is what will + know the job's declared secrets and pass them in -- this parameter exists now so + `build_event_record`/`build_result_receipt` never need to change shape when that wiring + lands. + 2. The kit's one cleaner, `providers.agent._redact_text`, all three of its steps: a URL loses + its userinfo, query and fragment, and keys, key blocks and named values are masked by + their shape. It logs nothing itself; one line names what it removed, on the cleaner's + own logger (not one of the loggers `MaskSecrets` filters, so the filter never re-enters), + and only when it changed the text. + 3. URL userinfo (`_USERINFO_PATTERN`/`_mask_userinfo`, above), for whatever step 2 left: a + password is masked and the username kept; a bare token/username-only userinfo (no `:`) + is masked outright, since that shape is how a bearer token appears, not a username. + + Step 2 removes a URL's userinfo outright, so `https://user:pw@host` now comes out as + `https://host`, not `https://user:***@host`. """ - redacted = _USERINFO_PATTERN.sub(_mask_userinfo, value) + redacted = value for secret in extra_secret_values: if len(secret) >= _SHORTEST_REDACTABLE_SECRET: redacted = redacted.replace(secret, "***") - return redacted + refused: list[str] = [] + hits: list[str] = [] + cleaned = _redact_text(redacted, refused, hits) + if (refused or hits) and cleaned != redacted: + _log_redacted(refused, hits) + return _USERINFO_PATTERN.sub(_mask_userinfo, cleaned) # ================================================================================================= diff --git a/src/fi/alk/harness/scenario_source.py b/src/fi/alk/harness/scenario_source.py index b4398a473..63361edb2 100644 --- a/src/fi/alk/harness/scenario_source.py +++ b/src/fi/alk/harness/scenario_source.py @@ -276,6 +276,12 @@ class _CompiledScenario: # same document and sent at pre-allocation, so a call can be read on the platform without the # scenario file beside it. Presentation only: nothing in the scheduler looks at it. presented: dict[str, Any] = field(default_factory=dict) + # The fields `presented` is built from, uncut (`_presented(..., fields=True)`). The provision + # payload cleans them and builds its record from them (TH-9500 cr2 M1, cr3 L4): `persona_of` + # cuts `name`, `role` and `scenario_name` at 255 characters and rewrites the name it shows, + # and a key cut or rewritten there no longer has a key's shape. Kept uncleaned, so loading a + # scenario cleans and logs nothing; `presented` stays as written, since the judge reads it. + presented_fields: dict[str, Any] = field(default_factory=dict) source_scenario_key: str | None = None # One correct way through the call: handed to reference checks and shown to the judge. solution: tuple[dict[str, Any], ...] = () @@ -313,12 +319,18 @@ def _validate_subgoal_name(name: str, *, folder_name: str) -> None: ) -def _presented(body: dict[str, Any], *, scenario_key: str) -> dict[str, Any]: +def _presented( + body: dict[str, Any], *, scenario_key: str, fields: bool = False +) -> dict[str, Any]: """The persona record the platform stores for a scenario, built by the platform module's own helper rather than a second copy of it here, so both paths describe a person the same way. Presentation only, and never load-bearing: anything missing from the document is simply absent from what the platform shows, and a document this cannot read still runs. + + With `fields`, the fields `persona_of` would read, `scenario_key` among them, uncut and as + written. They are the same keys `body` has, so a record built from them, cleaned, is + `_presented(cleaned, scenario_key=...)` (TH-9500 cr3 L4). """ from types import SimpleNamespace @@ -334,17 +346,17 @@ def _presented(body: dict[str, Any], *, scenario_key: str) -> dict[str, Any]: use_case = scenario_key.replace("-", " ").replace("_", " ") use_case = use_case[:1].upper() + use_case[1:] + read = { + "persona": body.get("persona") or {}, + "name": str(body.get("name") or ""), + "instruction": str(body.get("instruction") or ""), + "tests": str(body.get("tests") or ""), + "use_case": use_case, + } + if fields: + return {**read, "scenario_key": scenario_key} try: - return persona_of( - SimpleNamespace( - persona=body.get("persona") or {}, - name=str(body.get("name") or ""), - scenario_key=scenario_key, - instruction=str(body.get("instruction") or ""), - tests=str(body.get("tests") or ""), - use_case=use_case, - ) - ) + return persona_of(SimpleNamespace(**read, scenario_key=scenario_key)) except Exception: # noqa: BLE001 - a scenario never fails to run over how it is displayed return {} @@ -558,6 +570,7 @@ def _load_one( setup=setup, ready=ready, presented=_presented(body, scenario_key=scenario_key), + presented_fields=_presented(body, scenario_key=scenario_key, fields=True), solution=tuple(solution), ) @@ -935,14 +948,29 @@ def _provision_payload( `scenario.json` for scheduler-facing fields only, and the cost was a platform that could show a call but not who was on it or what they came for. """ + + def cleaned(scenario: Any) -> dict[str, Any]: + # `persona_of` over the cleaned fields (cr2 M1), here at the send, so its log line is + # the send's (cr3 L4). Cleaned again after: the shown name's `-` and `_` become spaces, + # which can give a key back its shape (`call_AKIA…` shows as `Call AKIA…`, cr3 M4). + read = dict(scenario.presented_fields) + if not read: + # No document was read (or it couldn't be): send the bare key, never a record + # `persona_of` would make up from nothing ("caller", "Scenario"). + return {} + scenario_key = str(read.pop("scenario_key", "") or "") + return _redact(_presented(_redact(read), scenario_key=scenario_key)) + payload: dict[str, Any] = { "operation": "provision", "name": run_name, # Everything the serializer accepts, where the document had it: name, role, situation, # outcome and the persona itself. `scenario_key` is set last because it is the one field the # platform matches on and it must be this scenario's, whatever the persona record says. + # The record is cleaned first (TH-9500): it is free text from the customer's own files, + # and this is the one place it leaves for the platform. "personas": [ - {**scenario.presented, "scenario_key": scenario.scenario_key} + {**cleaned(scenario), "scenario_key": scenario.scenario_key} for scenario in scenarios ], } diff --git a/src/fi/alk/harness/session.py b/src/fi/alk/harness/session.py index 739f04ba1..ffb364dee 100644 --- a/src/fi/alk/harness/session.py +++ b/src/fi/alk/harness/session.py @@ -25,6 +25,7 @@ from dataclasses import dataclass, field from typing import Any +from ..providers.agent import _redact from . import spend from .live import Channel, carrying, folded_in from .backends import ( @@ -385,7 +386,7 @@ def _events(self, received: Any, turn: Turn) -> list[Event]: events = [ Event( RESULT, - text=_shown(received.text), + text=_shown(_redact(received.text)), detail={ "is_error": received.is_error, "call_id": received.id, diff --git a/src/fi/alk/providers/agent.py b/src/fi/alk/providers/agent.py index 753df1e0b..cdffe8054 100644 --- a/src/fi/alk/providers/agent.py +++ b/src/fi/alk/providers/agent.py @@ -8,6 +8,7 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationError, model_validator +from . import secret_shapes from .setup_hash import compute_setup_hash logger = logging.getLogger(__name__) @@ -244,16 +245,29 @@ def _authored_model(value: Any) -> dict[str, Any] | None: return {"provider": provider, "name": name, "temperature": None, "max_tokens": None} -def _redact(value: Any, key: str = "") -> Any: +def _redact(value: Any, key: str = "", hits: list[str] | None = None) -> Any: + """`value` cleaned, with one log line when anything was masked. `hits` are kinds the caller + already masked in `value` before handing it here (chat's line-start mask, `_line_starts_joined`), + so they go on the same line.""" refused: list[str] = [] - redacted = _redact_in(value, key, refused) - if refused: - # Once per document, not per string: a crafted contract can hold half a million of them. - logger.warning("a URL was redacted whole: class=%s", refused[0]) + found = list(hits or ()) + redacted = _redact_in(value, key, refused, found) + if (refused or found) and (redacted != value or hits): + # Once per call, not per string: a crafted contract can hold half a million of them. + _log_redacted(refused, found) return redacted -def _redact_in(value: Any, key: str, refused: list[str]) -> Any: +def _log_redacted(refused: list[str], hits: list[str]) -> None: + """The cleaner's one line: the URL class it refused and the kinds of secret it masked, never + the text.""" + parts = [f"url={refused[0]}"] if refused else [] + if hits: + parts.append("kinds=" + ",".join(sorted(set(hits)))) + logger.warning("secrets redacted: %s", "; ".join(parts)) + + +def _redact_in(value: Any, key: str, refused: list[str], hits: list[str]) -> Any: lowered = key.lower() if any(marker in lowered for marker in _REDACTION_MARKERS): return _REDACTED @@ -263,13 +277,13 @@ def _redact_in(value: Any, key: str, refused: list[str]) -> Any: return _REDACTED if isinstance(value, Mapping): return { - str(item_key): _redact_in(item, str(item_key), refused) + str(item_key): _redact_in(item, str(item_key), refused, hits) for item_key, item in value.items() } if isinstance(value, list): - return [_redact_in(item, key, refused) for item in value] + return [_redact_in(item, key, refused, hits) for item in value] if isinstance(value, str): - return _redact_text(value, refused) + return _redact_text(value, refused, hits) return value @@ -285,27 +299,75 @@ def _redact_in(value: Any, key: str, refused: list[str]) -> Any: _QUERY_OR_FRAGMENT = re.compile(r"[?#]") -def _redact_text(value: str, refused: list[str]) -> str: - """Drop every URL's userinfo, query and fragment, and keep the rest of the text as written. +def _redact_text(value: str, refused: list[str], hits: list[str] | None = None) -> str: + """Clean one string in three steps (TH-9500), and keep the rest of the text as written. + + 1. The leftover rules, on the raw value: a password broken by whitespace, ?, # or /, a + userinfo holding an @ before the password's / (rule 1b), and a query after a break + (`secret_shapes.leftovers`). A value that is one URL is joined first (`_one_url`). + 2. TH-9491's URL step: drop every URL's userinfo, query and fragment. A URL runs from its + scheme to the next whitespace. Within one whitespace-free token, a later :// that starts + before the current URL's own ? or # begins a new URL; once a URL has a ? or #, the rest + of the token is its query or fragment. A key in a URL's host makes the whole URL + [redacted], matched before urlsplit lowers the host. + 3. The key patterns, on step 2's output (`secret_shapes.find`). - A URL runs from its scheme to the next whitespace. Within one whitespace-free token, a later - :// that starts before the current URL's own ? or # begins a new URL; once a URL has a ? or - #, the rest of the token is its query or fragment. + The kinds masked are added to `hits`; the caller logs them, so this logs nothing. """ - if "://" not in value: - return value - if _LEADING_SCHEME.match(value): - joined = value.translate(_URL_BREAKS) - if not _SPACE.search(joined): - # A field that is one URL, perhaps broken by a tab or a line break: one token. - return _redact_token(joined, refused) + # Both 600-character tests are made on the raw value: step 2 can shorten it (r1 M1). + cut = secret_shapes.is_cut(value) + joined = _one_url(value, hits) + if joined is not None: + # A value that is one URL runs step 1 on its joined token, as chat's `_clean` does, so + # the whole text agrees with the stream and rule 1b runs once (addendum M3, M-A): on the + # raw value, rule 1 would take a password broken by a line break up to its first @. + value = secret_shapes.leftovers(joined, hits) + value = _redact_token(value, refused, hits, secret_shapes.may_hold_key(value)) + return secret_shapes.mask(value, secret_shapes.find(value, cut), hits) + value = secret_shapes.leftovers(value, hits) + if "://" in value: + value = _redact_urls(value, refused, hits, secret_shapes.may_hold_key(value)) + return secret_shapes.mask(value, secret_shapes.find(value, cut), hits) + + +def _one_url(value: str, hits: list[str] | None) -> str | None: + """The one token a field that is one URL, perhaps broken by a tab or a line break, is + cleaned as; None when it is not one URL.""" + if not _LEADING_SCHEME.match(value): + return None + if _SPACE.search(value.translate(_URL_BREAKS)): + return None + return _line_starts_joined(value, hits) + + +def _line_starts_joined(value: str, hits: list[str] | None) -> str: + """`value` with its tabs and line breaks removed, as a URL joins, after the keys and named + values that begin a line or follow a tab are masked. Once joined, such a key would follow the + URL's last character, where no pattern starts (r2 M1). Chat's text before prose is joined here + too, so its stream masks what the whole text masks (change review r2, cr2 H1).""" + starts = secret_shapes.line_starts(value) + if starts: + # A line's first word is never inside a URL's host. + value = secret_shapes.mask(value, starts, hits, widen=False) + return value.translate(_URL_BREAKS) + + +def _redact_urls( + value: str, refused: list[str], hits: list[str] | None, keys: bool +) -> str: + # Step 1 can leave one URL where the raw value had a space: rule 1's password broken by one. + joined = _one_url(value, hits) + if joined is not None: + return _redact_token(joined, refused, hits, keys) return "".join( - part if not part or part.isspace() else _redact_token(part, refused) + part if not part or part.isspace() else _redact_token(part, refused, hits, keys) for part in _SPACE.split(value) ) -def _redact_token(token: str, refused: list[str]) -> str: +def _redact_token( + token: str, refused: list[str], hits: list[str] | None = None, keys: bool = False +) -> str: starts: list[tuple[int, bool]] = [] for match in _START.finditer(token): # The scheme is the run from its first letter; the text before that letter stays. @@ -326,7 +388,14 @@ def _redact_token(token: str, refused: list[str]) -> str: following = starts[index + 1][0] if index + 1 < len(starts) else len(token) end = len(token) if query_at < following else following piece = token[start:end] - if schemeless: + kind = secret_shapes.host_key(piece) if keys else None + if kind is not None: + # A key in the host: the whole URL goes, so step 3 never writes inside a host and a + # second pass changes nothing (r2 L3). + out.append(_REDACTED) + if hits is not None: + hits.append(kind) + elif schemeless: # "${SCHEME}://user:pw@host": parsed with a stand-in scheme, written back without it. cleaned = _redact_url("x" + piece, refused) out.append(cleaned[1:] if cleaned.startswith("x://") else cleaned) diff --git a/src/fi/alk/providers/secret_shapes.py b/src/fi/alk/providers/secret_shapes.py new file mode 100644 index 000000000..a923ceb98 --- /dev/null +++ b/src/fi/alk/providers/secret_shapes.py @@ -0,0 +1,1359 @@ +"""Secrets found by their shape, for the kit's one cleaner (TH-9500, design page v2.4). + +`providers.agent._redact_text` cleans a string in three steps: the two leftover rules here, on +the raw value; TH-9491's URL step; then the key patterns here (`find`), on that step's output. +Hosted chat sends its reply in pieces through the same cleaner, and `ChatHold` says where a +piece may end, so no rule here is split between two pieces. + +Nothing here does I/O, and nothing is imported but `re` and `bisect`. The kit's own pattern list +(`fi/evals/guardrails/scanners/secrets.py`) costs about 0.25 s to import and belongs to the evals +package, so only the drift test reads it, to tie the prefix patterns here to it. +""" + +import re +from bisect import bisect_left + +REDACTED = "[redacted]" + +# A key's own characters. A literal that follows one of them is inside a longer run, so no +# pattern starts there: each run is tried once, which keeps every scan linear. +_KEY = r"A-Za-z0-9_\-" +# The 600-character cut (`session._shown`) ends a text with this. +_AT_CUT = r"(?=\.\.\.\Z)" +_CUT_LENGTH = 600 + + +def _start(literal: str) -> str: + escaped = re.escape(literal) + # The literal comes first, so the regex engine's literal search finds each start. + return f"{escaped}(? str: + # A letter and a digit, or 16 or more characters. Read from the value's start and bounded + # at 16, so a name whose value fails is dropped by the engine, not by a Python check. + def test(char: str) -> str: + return rf"(?:(?={char}{{16}})|(?={char}{{0,15}}?{letter})(?={char}{{0,15}}?\d))" + + double, single, back = r'[^"\n]', r"[^'\n]", r"[^`\n]" + # A value already masked fails at once, so a second pass costs little. + return ( + r"(?![\"'`]?\[redacted\])" + r'(?:(?P")' + + test(double) + + r"|(?P')" + + test(single) + + r"|(?P`)" + + test(back) + + "|" + + test(_PLAIN_VALUE) + + ")" + ) + + +def _per_form(start: str) -> dict[str, re.Pattern[str]]: + # One regex per name, on the lower-cased text, each opening with its name, so the engine's + # literal search finds it as str.find would: a run of a name's own first letter never + # starts an attempt. `start` is a lookbehind on the name that says where it may sit. + tail = r"(?!::)" + _TAIL + _value_start("[a-z]") + return { + form: re.compile(re.escape(form) + start.format(re.escape(form)) + tail) + for form in _FORMS + } + + +# A name followed by "::" is code, not a name (Token::new(42), r1 L2). +# Not right after a letter or digit: the whole identifier, or its end after "_" or "-". "pass" +# only ever follows "_" or "-". +_NAMED = _per_form(r"(? bool: + """False when `text` holds no key literal at all, so no URL's host needs checking.""" + return any(root in text for root in _LITERAL_ROOTS) + + +_HEADER = "-----BEGIN " + + +# ---------------------------------------------------------------------------------- step 1 +# Rule 1: from scheme://user: to the next @, when the password is broken by whitespace, ?, # +# or / (TH-9491's URL step stops a URL at each). Its userinfo is removed when the text after +# "user:" doesn't begin with a port, holds at most one run of whitespace with text on both +# sides, and is at most 64 characters on each side of that run. +_PORT = ( + r"(?:\d+|\$?\{[^{}\s]*\}|\$[A-Za-z_][A-Za-z0-9_]*|%[A-Za-z_][A-Za-z0-9_]*%|<[^<>\s]*>" + r"|[A-Z][A-Z0-9_]*|[Pp][Oo][Rr][Tt])(?=[^A-Za-z0-9_\-@]|\Z)" +) +_HALF = r"[^\s@]{1,64}" +# A line break, after up to 4 spaces or tabs, with up to 16 characters of indent; or up to 4 +# spaces, tabs or NBSPs (r3 H2, r4 L3). +_RUN = r"(?:[ \t]{0,4}\r?\n[ \t]{0,16}|[ \t\xa0]{1,4})" +_SCHEME = r"(?\]}|]|\Z))" +) +_RULE1B = re.compile( + _SCHEME + # The port test (M1): with no : before the authority's first @, the text after its last : + # must not read as a port, so admin@localhost:8080/bob@example.com keeps its host. + + rf"(?![^\s/?#:@]{{0,129}}@[^\s/?#]{{0,194}}:{_PORT}/)" + + r"(" + # The authority holds a :, and it comes before any [ (M-C): admin@[::1]/bob@x keeps its host. + + r"(?=[^\s/?#\[]{0,129}:)" + # At most 64 characters before its first : or @. + + r"(?=[^\s/?#:]{0,64}[:@])" + # Up to the authority's last @, then at most 64 to the /, then 1-64 to the path's @. + + r"[^\s/?#\[\]]{0,129}@[^\s/?#@]{0,64}/[^\s/?#@]{1,64}@" + + r")" + + rf"(?={_RULE1B_HOST})" +) + +# Rule 2: the piece after an address piece loses its query values when it starts with ?, & or +# #, or whatever it starts with when the address's query ends in & or ?. +# The lookahead finds the piece's :// once (a lookahead never retries), so a long piece with +# many :// is scanned a bounded number of times. +_THEN_PIECE = re.compile(r"(? list[tuple[int, int]]: + """The userinfo rules 1 and 1b remove from `text`: each span runs from the user to its @. + + One sorted list, with spans that overlap or touch merged, since `remove` needs them in order + and apart (addendum M4): a://u:x://v:p@h/x@host.example is both rules' at once.""" + if "://" not in text or "@" not in text: + return [] + spans = [match.span(1) for match in _RULE1.finditer(text)] + rule1b(text) + spans.sort() + merged: list[tuple[int, int]] = [] + for start, end in spans: + if merged and start <= merged[-1][1]: + merged[-1] = (merged[-1][0], max(merged[-1][1], end)) + else: + merged.append((start, end)) + return merged + + +def rule1b(text: str) -> list[tuple[int, int]]: + """The userinfo rule 1b removes from `text`, up to the path's @. Its own function, so the + speed test can time the cleaner without it.""" + return [match.span(1) for match in _RULE1B.finditer(text)] + + +def rule2(text: str) -> list[tuple[int, int]]: + """The query values rule 2 empties in `text`: the piece after an address piece.""" + # It needs a ?, & or # somewhere: a query ends in one, or the next piece starts with one. + if "://" not in text or ("?" not in text and "&" not in text and "#" not in text): + return [] + spans: list[tuple[int, int]] = [] + for match in _THEN_PIECE.finditer(text): + address = match.group(1) + if ( + address[-1] in "&?" + and "?" not in address + and text[match.end()] not in "?&#" + ): + continue + piece = _PIECE.match(text, match.end()) + start = piece.start() + for param in _PARAM.finditer(piece.group()): + lead, name, value = param.groups() + # An empty value or the marker never matches, so cleaning twice changes nothing. + if not value or value == REDACTED: + continue + if lead == "&" and name.split(";")[0] in _ENTITIES: + continue + spans.append((start + param.start(3), start + param.end(3))) + return spans + + +def remove(text: str, spans: list[tuple[int, int]]) -> str: + """`text` without `spans`, which are in order and don't overlap.""" + out, at = [], 0 + for start, end in spans: + out.append(text[at:start]) + at = end + out.append(text[at:]) + return "".join(out) + + +def leftovers(value: str, hits: list[str] | None = None) -> str: + """Step 1: rule 1, then rule 2 on what rule 1 leaves.""" + spans = rule1(value) + if spans: + value = remove(value, spans) + if hits is not None: + hits.append("rule1") + spans = rule2(value) + if spans: + value = remove(value, spans) + if hits is not None: + hits.append("rule2") + return value + + +# ---------------------------------------------------------------------------------- step 3 +def is_cut(value: str) -> bool: + """Whether `value` is a text cut at 600 characters: both the length and the "..." (r4 L1). + Tested on the raw value, before step 1, so a URL query that step 2 drops before a key body + can't move it (r1 M1).""" + return len(value) == _CUT_LENGTH and value.endswith("...") + + +def find(text: str, cut: bool = False) -> list[tuple[int, int, str]]: + """Step 3: what to mask in `text`, as (start, end, kind). + + Keys by prefix or shape, key blocks, tokens in a URL's path and named values. With `cut`, + the raw value was a 600-character cut, so a key block with no END runs to the end and a key + cut short at the end is masked to it. + + Literals and names are looked for with `in` first (names in the lower-cased text), so text + with neither runs no pattern regex. + """ + low = text.translate(_ASCII_LOWER) + names = any(root in low for root in _NAME_ROOTS) + if not names and not any(root in text for root in _LITERAL_ROOTS): + return [] + literals = [literal for literal in _KEY_LITERALS if literal in text] + paths = [path for path in _PATHS if path[1] in text] + header = _HEADER in text + telegram = ":AA" in text + jwt = "eyJ" in text + guards = [guard for guard in _GUARDS if guard[1] in low] if names else [] + spans: list[tuple[int, int, str]] = [] + for kind, literal, pattern, cut_pattern in _PREFIXES: + if literal in literals: + spans.extend((m.start(), m.end(), kind) for m in pattern.finditer(text)) + if cut: + spans.extend( + (m.start(), m.end(), "cut_" + kind) + for m in cut_pattern.finditer(text) + ) + for kind, _, pattern, cut_pattern in paths: + spans.extend((m.start(1), m.end(1), kind) for m in pattern.finditer(text)) + if cut: + spans.extend( + (m.start(), m.end(), "cut_" + kind) for m in cut_pattern.finditer(text) + ) + if telegram: + _telegram(text, _TELEGRAM, "telegram", spans) + if cut: + _telegram(text, _TELEGRAM_CUT, "cut_telegram", spans) + if jwt: + spans.extend((m.start(), m.end(), "jwt") for m in _JWT.finditer(text)) + for kind, _, pattern in guards: + spans.extend((m.start(1), m.end(1), kind) for m in pattern.finditer(low)) + if header: + spans.extend((start, end, "private_key") for start, end in _blocks(text, cut)) + if names: + _named(text, low, spans) + return spans + + +def line_starts(text: str) -> list[tuple[int, int, str]]: + """Keys and named values that begin a line or follow a tab in `text`. + + For a value TH-9491 cleans as one URL: it joins the value's lines, which puts each line's + first word right after the URL's last character, where no pattern starts (r2 M1, r3 M7). + """ + spans: list[tuple[int, int, str]] = [] + for kind, literal, pattern, _ in _PREFIXES: + if literal in text: + spans.extend( + (m.start(), m.end(), kind) + for m in pattern.finditer(text) + if m.start() and text[m.start() - 1] in "\n\r\t" + ) + if ":AA" in text: + found: list[tuple[int, int, str]] = [] + _telegram(text, _TELEGRAM, "telegram", found) + spans.extend( + span for span in found if span[0] and text[span[0] - 1] in "\n\r\t" + ) + if "eyJ" in text: + spans.extend( + (m.start(), m.end(), "jwt") + for m in _JWT.finditer(text) + if m.start() and text[m.start() - 1] in "\n\r\t" + ) + low = text.translate(_ASCII_LOWER) + if any(root in low for root in _NAME_ROOTS): + _named(text, low, spans, at_line=True) + return spans + + +def mask( + text: str, + spans: list[tuple[int, int, str]], + hits: list[str] | None = None, + widen: bool = True, +) -> str: + """`text` with each span replaced by the marker, overlapping spans as one. + + A span that touches a URL's host widens to that whole URL: step 3 never writes inside a + host, so a second pass changes nothing (r2 L3).""" + if not spans: + return text + if widen and "://" in text: + widened = [] + for start, end, kind in spans: + url = _url_around(text, start, end) + widened.append((url[0], url[1], kind) if url else (start, end, kind)) + spans = widened + spans.sort() + out: list[str] = [] + kinds: set[str] = set() + at, begin, finish, group = 0, -1, -1, [] + for start, end, kind in spans + [(len(text) + 1, 0, "")]: + if start < finish: + # Overlapping spans are masked as one. + finish = max(finish, end) + group.append(kind) + continue + if begin >= 0: + if finish - begin != len(REDACTED) or text[begin:finish] != REDACTED: + kinds.update(group) + out.append(text[at:begin]) + out.append(REDACTED) + at = finish + begin, finish, group = start, end, [kind] + out.append(text[at:]) + if hits is not None: + hits.extend(sorted(kinds)) + return "".join(out) + + +def host_key(url: str) -> str | None: + """The kind of a key in `url`'s host, or None. Matched on the URL as written, before + urlsplit lowers the host, so an upper-case key is still found (r3 M7).""" + found = _NETLOC.search(url) + if found is None: + return None + host = found.group(1).rpartition("@")[2] + if not _HOST_LITERAL.search(host): + return None + for kind, literal, pattern, _ in _PREFIXES: + if literal in host and pattern.search(host): + return kind + if ":AA" in host: + found_tokens: list[tuple[int, int, str]] = [] + _telegram(host, _TELEGRAM, "telegram", found_tokens) + if found_tokens: + return "telegram" + if "eyJ" in host and _JWT.search(host): + return "jwt" + return None + + +_NETLOC = re.compile(r"://([^/?#]*)") +_HOST_STOP = re.compile(r"[/?#\s]") +_SCHEME_RUN_BEFORE = re.compile(r"[A-Za-z0-9+.\-]*\Z") +# A URL runs to the next whitespace or the next URL's ://, whichever comes first. +_URL_END = re.compile(r"\s|://") +# A host is looked for this far back from a match, so the search is bounded. +_HOST_REACH = 256 +# A name in a URL's host followed by a port, as in http://auth-token:8080/v1 (cr1 M1). +_HOST_PORT = re.compile(r":\d+(?=[/?#\s]|\Z)") + + +def _host_start(text: str, start: int) -> int: + """Where the host that `start` sits in begins, after its ://, or -1 when `start` is in no + URL's host. Looked for at most _HOST_REACH characters back.""" + scheme_end = text.rfind("://", max(0, start - _HOST_REACH), start) + if scheme_end < 0 or _HOST_STOP.search(text, scheme_end + 3, start): + return -1 + return scheme_end + 3 + + +def _url_around(text: str, start: int, end: int) -> tuple[int, int] | None: + """The span of the URL whose host [start, end) touches, or None.""" + host = _host_start(text, start) + if host < 0: + return None + scheme_end = host - 3 + stop = _HOST_STOP.search(text, start) + host_end = stop.start() if stop else len(text) + at = text.rfind("@", host, host_end) + if at >= 0: + host = at + 1 + if end <= host or start >= host_end: + return None + scheme = _SCHEME_RUN_BEFORE.search(text, max(0, scheme_end - 64), scheme_end) + begin = scheme.start() if scheme else scheme_end + # One search forward for the URL's end, and the next scheme's run looked for at most 64 + # back from its ://, as above: searched from every span instead, both were quadratic in a + # long run with no whitespace (cr1 H1). + stop = _URL_END.search(text, host_end) + finish = stop.start() if stop else len(text) + if stop and stop.group() == "://": + scheme = _SCHEME_RUN_BEFORE.search(text, max(host_end, finish - 64), finish) + finish = scheme.start() if scheme else finish + return begin, finish + + +def _telegram( + text: str, pattern: re.Pattern[str], kind: str, spans: list[tuple[int, int, str]] +) -> None: + for match in pattern.finditer(text): + digits = _BOT_ID.search(text[max(0, match.start() - 13) : match.start()]) + if digits: + spans.append((match.start() - len(digits.group(1)), match.end(), kind)) + + +def _blocks(text: str, cut: bool) -> list[tuple[int, int]]: + spans: list[tuple[int, int]] = [] + headers = list(_PEM.finditer(text)) + end_at, end = -2, None + done = 0 + for index, header in enumerate(headers): + if header.start() < done: + continue + # The next END is found once and reused for every header before it (r3 M6). + if end_at != -1 and end_at < header.end(): + end = _PEM_END.search(text, header.end()) + end_at = end.start() if end else -1 + following = ( + headers[index + 1].start() if index + 1 < len(headers) else len(text) + 1 + ) + if end is not None and header.end() <= end_at < following: + stop = end.end() + elif cut: + # Cut at 600 with no END: the block runs to the end of the text (r4 M6). + stop = len(text) + else: + stop = _key_lines_end(text, header.end()) + spans.append((header.start(), stop)) + done = stop + return spans + + +def _key_lines_end(text: str, start: int) -> int: + """Where a block with no END stops: after the key lines that follow its header, at the + first line that isn't one (r3 M5).""" + stop = start + rest = text[start : start + 1] + line = start if rest in ("", "\n", "\r") else _key_line_end(text, start) + if line < 0: + return stop + stop = line + newline = text.find("\n", line) + while newline >= 0: + line = _key_line_end(text, newline + 1) + if line < 0: + break + stop = line + newline = text.find("\n", line) if line < len(text) else -1 + if newline > line + 1: + # Something other than a \r came before the line break. + break + return stop + + +def _key_line_end(text: str, start: int) -> int: + """The end of the key line at `start` (without its \\r), or -1 if it isn't one. The last + line may end in the cut's "...".""" + line = _KEY_LINE.match(text, start) + if line is None or line.end() == start: + return -1 + end = line.end() + if end + 3 == len(text) and text.startswith("...", end): + return len(text) + carriage = text[end : end + 1] == "\r" + if end + carriage == len(text) or text[end + carriage : end + carriage + 1] == "\n": + return end + return -1 + + +def _named( + text: str, low: str, spans: list[tuple[int, int, str]], at_line: bool = False +) -> None: + if at_line: + # A value never runs past its line, so names at line starts never overlap. + for form in _FORMS: + if form in low: + for match in _NAMED_AT_LINE[form].finditer(low): + _take(text, match, spans) + return + found: list[tuple[int, str, re.Match[str]]] = [] + runs = 0 + for form in _FORMS: + if form not in low: + continue + before = len(found) + found.extend((m.start(), form, m) for m in _NAMED[form].finditer(low)) + if form in _NAMED_CAMEL: + found.extend( + (m.start(), form, m) + for m in _NAMED_CAMEL[form].finditer(low) + if text[m.start()].isupper() + ) + runs += len(found) > before + if runs > 1: + found.sort(key=_first) + at = 0 + for start, form, match in found: + # A name inside a matched value is not a start (r4 M7): overlapping values stay linear. + if ( + start >= at + and _name_sits(text, start, form) + and not _host_name(text, start, form) + ): + at = _take(text, match, spans) + + +def _host_name(text: str, start: int, form: str) -> bool: + """Whether the name at `start` ends a URL's host name followed by its port, as a Compose or + Kubernetes service name does (http://vault-secret:8200/v1/kv, cr1 M1): its ":" opens no + value. The port is checked first, so most names never look back for a host.""" + return ( + _HOST_PORT.match(text, start + len(form)) is not None + and _host_start(text, start) >= 0 + ) + + +def _take(text: str, match: re.Match[str], spans: list[tuple[int, int, str]]) -> int: + """Masks the value a name's match opens, unless it is a reference. Returns its end.""" + begin = match.end() + # The value's opening quote is the last group matched, if it has one. + quote = text[begin - 1] if match.lastgroup else "" + stop = (_QUOTE_END[quote] if quote else _PLAIN_END).search(text, begin) + finish = stop.start() if stop else len(text) + if _secret_value(text[begin:finish], bool(quote)): + spans.append((begin, finish, "named")) + return finish + + +def _first(found: tuple[int, str, re.Match[str]]) -> int: + return found[0] + + +def _name_sits(text: str, start: int, form: str) -> bool: + """Whether the name at `start` sits in its identifier as its kind allows.""" + kind, capital = _FORMS[form] + before = text[start - 1] if start else "" + joined = before == "_" or before == "-" + whole = not before or not (before.isalnum() or joined) + if kind == "upper": + return joined and text.startswith("PASS", start) and _upper_before(text, start) + if kind == "exact": + return whole or joined + camel = ( + text[start].isupper() + and before.isalnum() + and (not before.isupper() or text[start + 1 : start + 2].islower()) + ) + if kind == "compact": + return text[start + capital].isupper() and (whole or camel) + if form == "token" and _identifier(text, start, start + 5) in _PAGINATION: + return False + return whole or joined or camel + + +def _identifier(text: str, start: int, end: int) -> str: + # Bounded: no pagination name is longer than 21 characters. + begin = start + while ( + begin > 0 + and start - begin < 17 + and (text[begin - 1].isalnum() or text[begin - 1] == "_") + ): + begin -= 1 + return text[begin:end] + + +def _upper_before(text: str, start: int) -> bool: + # The identifier before PASS holds no lower-case letter, looked for at most 64 back. + begin = start + while ( + begin > 0 + and start - begin < 64 + and (text[begin - 1].isalnum() or text[begin - 1] == "_") + ): + begin -= 1 + if text[begin].islower(): + return False + return True + + +def _secret_value(value: str, quoted: bool) -> bool: + """Whether a value that passed the length test is masked: never when it is a reference.""" + if value.isalnum(): + # Letters and digits alone are neither a reference nor code. + return True + if ( + value.startswith(_REFERENCE_STARTS) + or "os.environ" in value + or "process.env" in value + or "env(" in value + ): + return False + return quoted or _CODE.match(value) is None + + +# ---------------------------------------------------------------------------------- chat +# A key is held whole up to this many characters, the cleaner's _HOLD_HARD; past it the +# cleaner's own long-URL rule applies. A key block has no such bound (r4 L7). +_LONGEST_HELD_KEY = 16_384 +_SPACE_RUN = re.compile(r"\s+") +_SEAM_NAME = re.compile(_NAME_ALTERNATION) +_SEAM_GUARD = re.compile("|".join(_GUARD_NAMES)) +_TAIL_PATTERN = re.compile(_TAIL) +# Every beginning of a tail: while what follows a name is one, its match is still open. +_TAIL_OPEN = re.compile( + r"[\"'`]?[*_`]{0,4}(?:[ \t]{0,16}|\s{0,4})(?:[=:][*_`]{0,4}(?:[ \t]{0,16}|\s{0,4}))?" +) +_GUARD_TAIL = re.compile(r"[\s:=]+['\"]?") +_GUARD_TAIL_OPEN = re.compile(r"[\s:=]*") +_GUARD_VALUE_END = re.compile(r"[^A-Za-z0-9/+=]") +_PLAIN_START = re.compile(_PLAIN_VALUE) +_USER_END = re.compile(r"[\s/?#@:\[\]]") +# Every beginning of rule 1's text after "user:", while no @ has come: a CRLF's \r alone too, +# when its \n is still to come in the next delta (cr2 M3). +_RULE1_OPEN = re.compile(rf"[^\s@]{{0,64}}(?:{_RUN}[^\s@]{{0,64}}|[ \t]{{0,4}}\r)?") +_HEADER_OPEN = re.compile(r"-----BEGIN[A-Z0-9 ]{0,60}-{0,5}") +_SEAM_KEY = _HOST_LITERAL +_URL_MARK = re.compile("://") +_LONGEST_END = len("-----END " + "X" * 40 + "PRIVATE KEY BLOCK-----") +# How far back a delta's search reaches into the text before it: the longest name, guard name +# or literal, less one, so one split across two deltas is still found. +_REACH = ( + max(_LONGEST_NAME, len("azure_storage_account_key"), *map(len, _KEY_LITERALS)) - 1 +) + +# A named value's phases in ChatHold. +_FRAME, _QUOTED, _PLAIN = range(3) + + +def _piece_start(piece: "_Piece") -> int: + return piece.start + + +class _Piece: + """A whitespace-separated piece of the stream.""" + + __slots__ = ("start", "end", "address", "after", "extends", "query_end") + + def __init__(self, start: int, before: "_Piece | None") -> None: + self.start = start + self.end = -1 + self.address = False + # The address piece this one follows, if any: a candidate here holds from there. + self.after = before if before is not None and before.address else None + # Whether rule 2 reaches into this piece from the address piece before it. + self.extends = False + # For an address piece: its query ends in & or ?, so rule 2 reaches the next piece. + self.query_end = False + + +class ChatHold: + """Where hosted chat may cut its streamed reply, so no rule here is split between two + pieces (design page, "The hold rule" and "The seam"). + + `_DeltaCleaner` holds the text after its last cut and cleans each piece it lets out. Rule 1, + rule 2, a named value and a key block match across whitespace, so the cleaner asks here how + far it may go: never past its own cut, never into a match that is still open or has just + closed, and never into an address piece holding a candidate, or the piece after one. + + The work per delta is bounded by the delta's length plus a constant (r4 H2, r1 M7): the + state is kept between deltas, and only the new text and a short tail are searched. A + candidate's match is decided once, when what closes it arrives. Positions count from the + start of the prose stream; the held text is always the end of it. + """ + + def __init__(self) -> None: + # Set by the cleaner, whose `_learn` alone decides it. Before prose the cleaner never + # cuts, since a lone URL broken by a line break is one URL (r4 M2), so neither does this. + self.prose = False + self._end = 0 + # Just after the last whitespace: the cleaner's own cut. + self._space_end = 0 + self._pieces: list[_Piece] = [] + # Open candidates, by where each starts: named values, guard names, rule 1's + # scheme://user: and key headers still arriving. + self._names: dict[int, list] = {} + self._guards: dict[int, list] = {} + self._users: dict[int, list] = {} + self._headers: dict[int, int] = {} + # Named values and guard names that have closed, so the tail search doesn't take them + # again. + self._done: set[int] = set() + # Matches that have closed, as (start, end, hold start): no cut falls inside one, so a + # match that has just closed goes out whole (r3 M3). + self._closed: list[tuple[int, int, int]] = [] + # An open key block: [hold start, header start, where END is next looked for]. + self._block: list[int] | None = None + # A block no END can close, since a later header came first: it holds to the end. + self._forever: int | None = None + # Where a key in the last, unfinished piece holds from. + self._key: int | None = None + # The last "://" and "-----BEGIN" taken, so each is taken once. + self._url_seen = -1 + self._header_seen = -1 + + def cut(self, held: str, delta: str) -> int: + """The index into `held + delta` up to which the text may be sent.""" + if not self.prose: + return 0 + base = self._end - len(held) + old_end = self._end + self._end += len(delta) + whole = held + delta + self._forget(base) + if self._forever is not None: + return 0 + self._split(whole, base, old_end) + start = max(base, old_end - _REACH) + while True: + if self._block is not None: + closed = self._block_end(whole, base) + if closed < 0: + break + start = max(start, closed) + if not self._look(whole, base, start): + break + if self._forever is not None: + return 0 + self._follow(whole, base) + return self._limit(base) - base + + def shed_cut(self, held: str) -> int | None: + """Where the cleaner's flush past its soft bound may cut `held`. + + None when nothing here holds the text, so the cleaner sheds as before; 0 when nothing may + be sent yet. A key block is held whatever its length; a key, a named value and a match + that has closed, up to the hard bound.""" + if not self.prose: + return None + base = self._end - len(held) + holds = list(self._headers.values()) + if self._forever is not None: + holds.append(self._forever) + if self._block is not None: + holds.append(self._block[0]) + if not holds and self._key is not None and len(held) <= _LONGEST_HELD_KEY: + holds.append(self._key) + if len(held) <= _LONGEST_HELD_KEY: + # A named value longer than the soft bound is held as a key is (cr1 M3): shed past + # it, the cleaned text before the cut would mask the value only that far, and the + # rest would go out later with no name in front of it. A match that has closed is + # held too: one still held waits on the piece after its URL, and the branches would + # cut it at that URL's start, between its name and its value. + holds += [state[4] for state in self._names.values()] + holds += [state[4] for state in self._guards.values()] + holds += [hold for _, end, hold in self._closed if end > base] + if not holds: + return None + return max(0, min(holds) - base) + + # -- the stream --------------------------------------------------------------------------- + + def _forget(self, base: int) -> None: + # What the cleaner has sent or dropped since the last delta is no longer held. + self._closed = [span for span in self._closed if span[1] > base] + for found in (self._names, self._guards, self._users, self._headers): + for start in [start for start in found if start < base]: + del found[start] + self._done = {start for start in self._done if start >= base} + if self._block is not None and self._block[1] < base: + self._block = None + pieces = self._pieces + gone = 0 + while gone < len(pieces) - 1 and -1 < pieces[gone].end <= base: + gone += 1 + if gone: + del pieces[:gone] + if self._key is not None and self._key < base: + self._key = base + + def _split(self, whole: str, base: int, old_end: int) -> None: + """Brings the pieces up to date with the delta.""" + at = old_end + for space in _SPACE_RUN.finditer(whole, old_end - base): + begin = space.start() + base + if begin > at: + self._grow(whole, base, at, begin) + self._close(whole, base, begin) + self._space_end = at = space.end() + base + if at < self._end: + self._grow(whole, base, at, self._end) + + def _grow(self, whole: str, base: int, start: int, end: int) -> None: + pieces = self._pieces + if not pieces or pieces[-1].end != -1: + before = pieces[-1] if pieces else None + piece = _Piece(start, before) + if piece.after is not None: + piece.extends = before.query_end or whole[start - base] in "?&#" + pieces.append(piece) + piece = pieces[-1] + if not piece.address: + piece.address = ( + "://" in whole[max(piece.start, start - 2, base) - base : end - base] + ) + + def _address(self, whole: str, base: int, end: int) -> None: + """Marks the piece a rule 1 match ends in as an address piece (cr2 M2). A match that + crossed whitespace ends in a piece with no :// of its own (`ss@h:6379` in + `redis://:pa ss@h:6379`), which is the URL the whole text's rule 1 leaves, so rule 2 may + reach the piece after it and that piece holds from it.""" + pieces = self._pieces + # The last piece starting before `end`, found by bisection: a walk back from the end + # made one large delta with many such matches quadratic (cr3 M1). + index = bisect_left(pieces, end, key=_piece_start) - 1 + if index < 0: + return + piece = pieces[index] + if piece.address: + return + piece.address = True + if piece.end == -1: + return + # It closed in this delta, so what `_close` and `_grow` decided from it is decided again. + text = whole[max(piece.start, base) - base : piece.end - base] + piece.query_end = text[-1:] in ("&", "?") and "?" in text + if index + 1 < len(pieces) and base <= pieces[index + 1].start < self._end: + following = pieces[index + 1] + following.after = piece + following.extends = ( + piece.query_end or whole[following.start - base] in "?&#" + ) + if following.extends and following.end != -1: + self._closed.append((piece.start, following.end, piece.start)) + + def _close(self, whole: str, base: int, end: int) -> None: + pieces = self._pieces + if not pieces or pieces[-1].end != -1: + return + piece = pieces[-1] + piece.end = end + # A key's characters hold no whitespace: one in this piece has closed. + self._key = None + if piece.address: + text = whole[max(piece.start, base) - base : end - base] + piece.query_end = text[-1:] in ("&", "?") and "?" in text + if piece.extends: + # Rule 2 has matched: the address piece and this one go out together. + self._closed.append((piece.after.start, end, piece.after.start)) + + def _hold_of(self, at: int) -> int: + """Where a candidate at `at` holds from: its piece's start, or the start of the address + piece it is in or follows, so a URL's path and query stay whole (r2 H1, r4 M1).""" + for piece in reversed(self._pieces): + if piece.start <= at: + if piece.address: + return piece.start + if piece.after is not None: + return piece.after.start + return piece.start + return at + + # -- candidates --------------------------------------------------------------------------- + + def _look(self, whole: str, base: int, start: int) -> bool: + """Takes the candidates that start at `start` or later. True when a key header opened a + block: the text after it is then left to the block.""" + begin = start - base + stop = len(whole) + opened = False + found = whole.find("-----BEGIN", max(begin, self._header_seen + 1 - base)) + while found >= 0: + self._header_seen = found + base + header = _PEM.match(whole, found) + if header is not None: + self._block = [ + self._hold_of(found + base), + found + base, + header.end() + base, + ] + stop, opened = found, True + break + if _HEADER_OPEN.fullmatch(whole, found): + self._headers[found + base] = self._hold_of(found + base) + found = whole.find("-----BEGIN", found + 1) + self._look_urls(whole, base, begin, stop) + self._look_keys(whole, base, begin, stop) + self._look_names(whole, base, begin, stop) + return opened + + def _look_urls(self, whole: str, base: int, begin: int, stop: int) -> None: + for mark in _URL_MARK.finditer( + whole, max(begin, self._url_seen + 1 - base), stop + ): + at = mark.start() + self._url_seen = at + base + scheme = _SCHEME_RUN_BEFORE.search(whole, max(0, at - 64), at) + first = scheme.start() if scheme else at + if first < at and whole[first].isalpha(): + # Rule 1 may start here; its user runs to the first of _USER_END. + self._users[first + base] = [ + first + base, + at + 3 + base, + -1, + self._hold_of(at + base), + ] + + def _look_keys(self, whole: str, base: int, begin: int, stop: int) -> None: + pieces = self._pieces + if not pieces or pieces[-1].end != -1: + return + for key in _SEAM_KEY.finditer(whole, max(begin, pieces[-1].start - base), stop): + hold = self._hold_of(key.start() + base) + self._key = hold if self._key is None else min(self._key, hold) + + def _look_names(self, whole: str, base: int, begin: int, stop: int) -> None: + low = whole[begin:stop].translate(_ASCII_LOWER) + for pattern, found in ((_SEAM_NAME, self._names), (_SEAM_GUARD, self._guards)): + for name in pattern.finditer(low): + at = begin + name.start() + base + if at not in found and at not in self._done: + found[at] = [ + _FRAME, + at + len(name.group()), + -1, + "", + self._hold_of(at), + ] + + def _block_end(self, whole: str, base: int) -> int: + """Looks for the open block's END in the new text and a tail as long as END's longest + form. Returns where the block closed, or -1 while it is open.""" + block = self._block + start = max(block[2] - _LONGEST_END, block[1] + 1) - base + end = _PEM_END.search(whole, start) + limit = end.start() if end is not None else len(whole) + later = whole.find( + "-----BEGIN", max(start, self._header_seen + 1 - base), limit + ) + while later >= 0: + self._header_seen = later + base + if _PEM.match(whole, later) or _HEADER_OPEN.fullmatch(whole, later): + # Another header before any END: an END pairs with the nearest header before + # it (r4 L4), so none can close this block. It holds to the message's end. + self._forever = block[0] + return -1 + later = whole.find("-----BEGIN", later + 1, limit) + if end is None: + block[2] = self._end + return -1 + self._closed.append((block[1], end.end() + base, block[0])) + self._block = None + return end.end() + base + + def _follow(self, whole: str, base: int) -> None: + """Moves each open candidate on by what the delta brought, searching new text only.""" + for found, step in ( + (self._names, self._name_step), + (self._guards, self._guard_step), + ): + for at, state in list(found.items()): + closed = step(whole, base, at, state) + if closed is None: + continue + del found[at] + if closed > 0: + self._done.add(at) + self._closed.append((at, closed, state[4])) + for at, state in list(self._users.items()): + closed = self._user_step(whole, base, state) + if closed is None: + continue + del self._users[at] + if closed > 0: + self._closed.append((at, closed, state[3])) + self._address(whole, base, closed) + for at in list(self._headers): + if _PEM.match(whole, at - base) is not None: + hold = self._headers.pop(at) + if self._block is None: + self._block = [hold, at, at + 1] + else: + self._forever = self._block[0] + elif not _HEADER_OPEN.fullmatch(whole, at - base): + del self._headers[at] + + def _name_step(self, whole: str, base: int, at: int, state: list) -> int | None: + """None while the named value may still match; its end once it closes; 0 when it can't + match.""" + if state[0] == _FRAME: + low = whole[at - base : state[1] - base + _LONGEST_NAME].translate( + _ASCII_LOWER + ) + name = _SEAM_NAME.match(low) + if name is None: + return 0 + name_end = at - base + len(name.group()) + if _TAIL_OPEN.fullmatch(whole, name_end): + return None + if whole.startswith("::", name_end): + return 0 + tail = _TAIL_PATTERN.match(whole, name_end) + if tail is None: + return 0 + value = tail.end() + if value >= len(whole): + return None + if whole[value] in "\"'`": + state[0], state[2], state[3] = _QUOTED, value + 1 + base, whole[value] + elif _PLAIN_START.match(whole, value): + state[0], state[2] = _PLAIN, value + base + else: + return 0 + if state[0] == _QUOTED: + stop = _QUOTE_END[state[3]].search(whole, state[2] - base) + if stop is None: + state[2] = self._end + return None + closed = stop.end() if whole[stop.start()] == state[3] else stop.start() + return closed + base + stop = _PLAIN_END.search(whole, state[2] - base) + if stop is None: + state[2] = self._end + return None + return stop.start() + base + + def _guard_step(self, whole: str, base: int, at: int, state: list) -> int | None: + """As `_name_step`, for aws_secret_key and azure_storage_key's own names and tail.""" + if state[0] == _FRAME: + name_end = state[1] - base + if _GUARD_TAIL_OPEN.fullmatch(whole, name_end): + return None + tail = _GUARD_TAIL.match(whole, name_end) + if tail is None: + return 0 + if tail.end() >= len(whole): + return None + state[0], state[2] = _PLAIN, tail.end() + base + stop = _GUARD_VALUE_END.search(whole, state[2] - base) + if stop is None: + state[2] = self._end + return None + return stop.start() + base + + def _user_step(self, whole: str, base: int, state: list) -> int | None: + """Rule 1 from this scheme: None while it may still match, its end once it does, 0 when + it can't.""" + if state[2] < 0: + stop = _USER_END.search(whole, state[1] - base) + if stop is None: + state[1] = self._end + return None + if whole[stop.start()] != ":": + return 0 + state[1] = state[2] = stop.end() + base + at = whole.find("@", state[1] - base) + if at < 0: + state[1] = self._end + return None if _RULE1_OPEN.fullmatch(whole, state[2] - base) else 0 + match = _RULE1.match(whole, state[0] - base) + return match.end() + base if match is not None else 0 + + def _limit(self, base: int) -> int: + """The cleaner's own cut, moved back before each open candidate's hold start and out of + each closed match.""" + holds = [state[4] for state in self._names.values()] + holds += [state[4] for state in self._guards.values()] + holds += [state[3] for state in self._users.values() if state[2] >= 0] + holds += self._headers.values() + if self._block is not None: + holds.append(self._block[0]) + if self._key is not None: + holds.append(self._key) + if self._pieces: + last = self._pieces[-1] + if last.end != -1 and last.address: + # Rule 2 may yet reach the piece after it. + holds.append(last.start) + elif last.end == -1 and last.extends: + holds.append(last.after.start) + cut = min(self._space_end, *holds) if holds else self._space_end + moved = True + while moved: + moved = False + for start, end, hold in self._closed: + if start < cut < end and hold < cut: + cut, moved = hold, True + return max(cut, base) diff --git a/tests/harness/test_builder_chat.py b/tests/harness/test_builder_chat.py index 703539a07..12a1f7ff2 100644 --- a/tests/harness/test_builder_chat.py +++ b/tests/harness/test_builder_chat.py @@ -2707,3 +2707,1365 @@ def test_random_splits_never_send_a_credential_the_stripped_whole_drops(): if leaked: found.append((text, leaked)) assert found == [] + + +# -- TH-9500: the hold rule for secrets matched across whitespace (plan v1.1, C1-C4) -------------- +# Every key-shaped value is fake. The cases are the design reviewers' lists, copied as data. + +_GH = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8" +_SKP = "sk-proj-" + "FAKEfake0123456789abcdefFAKEfake0123456789ab" +_AK = "AKIA" + "FAKEFAKE12345678" +_XB = "xoxb-12345678901-12345678901-" + "FAKEfakeFAKEfakeFAKEfake" +_SLACK = "https://hooks.slack.com/services/" + "T00FAKE00/B00FAKE00/" + "FAKEfakeFAKEfakeFAKEfake" +_TG = "https://api.telegram.org/bot123456789:AAFAKEfakeFAKEfakeFAKEfakeFAKEfake_x/sendMessage" +_JWT = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJmYWtlIn0.FAKEsigFAKEsig" +_PEM = "-----BEGIN RSA PRIVATE KEY-----\nMIIEFAKEFAKEFAKE\nABCDEFFAKE==\n-----END RSA PRIVATE KEY-----" +# Round 1's cases (r1cases.py) and the addendum's three: round 2's 1,919 stream splits. +_ROUND1 = { + "page: ws in password": "https://bot:hun ter2@h/v1", + "page: lb in password": "see https://bot:hun\n ter2@h/v1 is the hook", + "page: tab in password": "see https://bot:hun\tter2@h/v1 is the hook", + "page: query next line": "see https://u:pw@h/x\n ?key=sk-1 and more", + "page: ?# in password": "https://bot:8812#x@db/", + "page: / in password": "https://bot:pa/ss@db", + "page: :// in password": "https://bot:pa://ss@db", + "bare ghp prose": f"Use {_GH} to push.", + "bare sk-proj prose": f"key is {_SKP}.", + "bare AKIA quoted": f'aws="{_AK}"', + "bare xoxb json": f'{{"t": "{_XB}"}}', + "slack hook": f"Post to {_SLACK} daily", + "telegram": f"Call {_TG} now", + "jwt prose": f"Bearer {_JWT}", + "pem": f"key:\n{_PEM}\nend", + "pem header only": "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAAFAKE\n(truncated)", + "assign pw": "password=hunter22", + "assign spaced": "token: s3cr3tvalue99", + "tab query": "see https://h/x\t?key=sk-1 more", + "mid-query break": "see https://h/x?a=1&\n key=sk-1 more", + "break before ? no indent": "see https://h/x\n?key=sk-1 more", + "nbsp query": "see https://h/x\xa0?key=sk-1 more", + "nbsp password": "https://bot:hun\xa0ter2@h/v1", + "H port+email": "Run at http://localhost:3000 and email support@acme.com", + "H query email": "https://host:8080/path?email=a@b.com", + "H npm scoped": "Install from https://registry.npmjs.org:443/@babel/core today", + "H medium": "Read https://medium.com/@user for more", + "H port then mention": "Open http://localhost:8080 and ping @oncall in Slack", + "H md amp": "See https://docs.x.com/a\n©=2024 Acme", + "H password required": "password: required", + "H token expires": "the token: expires hourly", + "H secret never": "secret: never tell anyone", + "H env": 'api_key=os.environ["OPENAI_API_KEY"]', + "H env placeholder": "Send the token: ${SLACK_BOT_TOKEN} in the header", + "H commit": "Fixed in 1ac329e16b0f2c9a8e7d6c5b4a3f2e1d0c9b8a7f", + "H uuid": "order 123e4567-e89b-12d3-a456-426614174000", + "H b64img": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", + "H order": "Order ID ORD-2024-000123 and SKU AKIAMODEL", + "H token count": "max_tokens: 4096 and token=streaming", + "a1": 'config "password": "Zq8wYv3Lp9" ok', + "a3": "use https://bot:hun ter2@h/v1 now", + "a4": f"key:\n{_PEM}\nend", +} +# The design page's chat-test list, each with trailing text. +_RUN = "/".join(["a" * 50] * 90) +_CHAT_CASES = { + "ghp in a long run": '{"x":"' + + _RUN[:3000] + + '","t":"' + + _GH + + '","y":"' + + _RUN[:2000] + + '"} done now', + "named value at its ': '": 'config "password": "Zq8wYv3Lp9" ok then more', + "rule 1 at its space": "use https://bot:hun ter2@h/v1 now and later", + "rule 1 at a break + 4": "see https://bot:hun\n ter2@h/v1 is the hook", + "block": f"key:\n{_PEM}\nend of message here", + "block in YAML": f"tls:\n key: |\n {_PEM.replace(chr(10), chr(10) + ' ')}\n ok: true", + "block CRLF": f"Here:\r\n{_PEM.replace(chr(10), chr(13) + chr(10))}\r\nKeep it safe.", + "block encrypted": "Here:\n-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: " + "AES-128-CBC,3F17F5316E2BAC89\n\nMIIEFAKEQUJDREVG\n-----END RSA PRIVATE KEY-----\nKeep it.", + "block one line": "Use this: -----BEGIN RSA PRIVATE KEY----- MIIEFAKEQUJDREVG " + "-----END RSA PRIVATE KEY----- then restart", + "login query": "Login at https://h.example/login?user=bob&password=Sunshine then stop", + "oauth code": "POST https://oauth2.googleapis.com/token?code=4/0AXfakeFAKEcode123&grant_type=x and wait", + "telegram path": f"Call {_TG}?chat_id=1 now please", + "a URL holding a name and a prefix": "Open https://h.example/cb/token?ref=sk-not-a-key&session=Qm9vZmFrZQ then go", + "rule 2 on the next line": "see https://h/x?a=1\n&token=Sunshine and more", + "named md label": "- **Password:** Zq8wYv3Lp9 then go on", + "named camel json": '{"clientSecret": "Zq8wYv3Lp9Kd2xT", "x": 1} done now', + "named env pass": "DB_PASS=Zq8wYv3Lp9\nSMTP_PASS=hunter22x9\nok", + "named 16 spaces": 'admin = "x"\npassword = "Zq8wYv3Lp9" ok', + "named kebab": "cloud.aws.credentials.secret-key: Zq8wYv3Lp9Kd2xT ok", + "named quoted with spaces": 'cfg "password": "correct horse battery 9" ok more', + "named break + 4": "password:\n Zq8wYv3Lp9 ok", + "conn string": "Use DefaultEndpointsProtocol=https;AccountName=acct;AccountKey=FAKEazureFAKEkey0123" + "456789abcdef==;EndpointSuffix=core.windows.net then go", + "azure key, spaced": "set AZURE_KEY = " + + ("FAKEazureFAKEkey0123456789abcdefFAKE+/" * 3)[:86] + + "== ok", +} +_PLANTED = [ + "hun ter2", + "ter2@", + "hunter22", + "Zq8wYv3Lp9", + "s3cr3tvalue99", + "MIIEFAKE", + "ABCDEFFAKE", + _GH, + _SKP, + "battery 9", + "FAKEazureFAKEkey0123", + "Sunshine", + "4/0AXfake", + "Qm9vZmFrZQ", + "AAFAKEfake", + "key=sk-1", + "FAKEfakeFAKEfakeFAKEfake", + "FAKEsigFAKEsig", + "QUJDREVG", +] + + +def _events_of(deltas: list[str]) -> list[str]: + from fi.alk.harness.hosted_chat_entrypoint import _DeltaCleaner + + cleaner = _DeltaCleaner() + events = [cleaner.feed(delta) for delta in deltas] + events.append(cleaner.flush()) + return [event for event in events if event] + + +def _removed_fragments(text: str, whole: str) -> set[str]: + """k1_stream's frags(): 6-character fragments of what the whole clean removes, less any + that also occur in what it keeps (r4 L8).""" + import difflib + + matcher = difflib.SequenceMatcher(None, text, whole, autojunk=False) + removed = [ + text[start:end] + for op, start, end, _, _ in matcher.get_opcodes() + if op in ("delete", "replace") and len(text[start:end].strip()) >= 6 + ] + fragments = { + piece[at : at + 6] for piece in removed for at in range(len(piece) - 5) + } + return {fragment for fragment in fragments if fragment not in whole} + + +def _split_problems(text: str, splits: list[list[str]]) -> list[str]: + """C1's three checks, for each way of splitting `text` into deltas: the joined events equal + one whole clean; no 6-character fragment of a removed string is in any event; no planted + secret the whole clean drops is in the joined events.""" + from fi.alk.providers.agent import _redact + + whole = _redact(text) + fragments = _removed_fragments(text, whole) + problems = [] + for deltas in splits: + events = _events_of(deltas) + joined = "".join(events) + if joined != whole: + problems.append(f"unequal at {[len(d) for d in deltas][:3]}") + elif any( + event[at : at + 6] in fragments + for event in events + for at in range(len(event) - 5) + ): + problems.append(f"a fragment at {[len(d) for d in deltas][:3]}") + elif any( + secret in text and secret not in whole and secret in joined + for secret in _PLANTED + ): + problems.append(f"a planted secret at {[len(d) for d in deltas][:3]}") + return problems + + +def _every_split(text: str) -> list[list[str]]: + return [[text[:at], text[at:]] for at in range(1, len(text))] + [list(text)] + + +def test_round_twos_1919_stream_splits_each_equal_one_whole_clean() -> None: + """C2 (W3): round 1's cases and the addendum's, every two-way split and char by char.""" + splits = {name: _every_split(text) for name, text in _ROUND1.items()} + assert sum(len(each) for each in splits.values()) == 1919 + + problems = { + name: _split_problems(_ROUND1[name], each) for name, each in splits.items() + } + + assert {name: found[:3] for name, found in problems.items() if found} == {} + + +@pytest.mark.parametrize("name", list(_CHAT_CASES)) +def test_each_chat_case_split_at_every_offset_equals_one_whole_clean(name: str) -> None: + """C1: every offset (sampled in the long run), and 1-, 3- and 7-character deltas.""" + text = _CHAT_CASES[name] + offsets = range(1, len(text)) if len(text) < 2000 else range(1, len(text), 7) + splits = [[text[:at], text[at:]] for at in offsets] + splits += [ + [text[at : at + size] for at in range(0, len(text), size)] for size in (1, 3, 7) + ] + + assert _split_problems(text, splits)[:3] == [] + + +def test_every_event_the_pump_sends_masks_a_secret_split_across_its_whitespace( + tmp_path, monkeypatch +): + """C1 through _pump_events: assistant_delta, tool_result and the turn's assistant_message + (the question is below). A named value split at its ': ' and a password at its space.""" + from fi.alk.providers.agent import _redact + + client = BuilderClient() + runtime = _runtime(tmp_path, client, monkeypatch) + queue: asyncio.Queue = asyncio.Queue() + said = 'Set "password": "Zq8wYv3Lp9" and use https://bot:hun ter2@h.example/v1 then go.' + colon, space = said.index(": ") + 1, said.index("hun ") + 4 + pieces = [ + said[:colon], + said[colon : colon + 1], + said[colon + 1 : space], + said[space:], + ] + for piece in pieces: + queue.put_nowait( + RuntimeEvent(kind="text", text=piece, detail={"partial": True}) + ) + queue.put_nowait( + RuntimeEvent(kind="tool", tool="mcp__x__Read", detail={"call_id": "r"}) + ) + queue.put_nowait( + RuntimeEvent( + kind="result", text=f".env:1: GITHUB_TOKEN={_GH}", detail={"call_id": "r"} + ) + ) + queue.put_nowait(None) + + asyncio.run( + runtime._pump_events( + queue, + assistant_message_id="a-1", + fallback_invocation_id="turn-1", + emitted_text=[], + turn_failures=[], + ) + ) + + shown = json.dumps(client.events) + assert "Zq8wYv3Lp9" not in shown and "ter2" not in shown and _GH[4:12] not in shown + assert "".join(_deltas(client)) == _redact(said) + [result] = [one["payload"] for one in client.events if one["kind"] == "tool_result"] + assert result["text"] == ".env:1: GITHUB_TOKEN=[redacted]" + + +def test_the_turns_assistant_message_masks_a_key_split_between_its_pieces( + tmp_path, monkeypatch +): + client = BuilderClient() + runtime = _runtime(tmp_path, client, monkeypatch) + runtime._published_fingerprint() + runtime.conversation = _Saying( + tmp_path, _texts(["Saved password: ", "Zq8wYv3Lp9 ", "for you."]) + ) + + _turn(runtime, {"content": "save it"}) + + assert "Zq8wYv3Lp9" not in json.dumps(client.events) + assert "".join(_deltas(client)) == "Saved password: [redacted] for you." + assert _reply(client) == "Saved password: [redacted] for you." + + +def test_a_question_the_model_asks_masks_a_named_value(tmp_path, monkeypatch): + client = BuilderClient() + runtime = _runtime(tmp_path, client, monkeypatch) + runtime.stopping.set() + + asyncio.run( + runtime._ask( + "AskUserQuestion", + { + "question": "Keep DB_PASSWORD=Zq8wYv3Lp9 ?", + "options": [f"use {_GH}", "drop it"], + }, + None, + ) + ) + + asked = next(one for one in client.events if one["kind"] == "question_requested") + assert asked["payload"] == { + "prompt": "Keep DB_PASSWORD=[redacted] ?", + "options": ["use [redacted]", "drop it"], + } + + +def _seconds(deltas: list[str]) -> float: + import time + + best = 9.0 + for _ in range(3): + start = time.perf_counter() + _events_of(deltas) + best = min(best, time.perf_counter() - start) + return best + + +def test_the_seams_work_per_delta_is_bounded_by_the_delta() -> None: + """C3 (r4 H2, r1 M7): each under 0.5 s, best of 3.""" + words = "The quick brown fox jumps over the lazy dog while the build runs. " + refuse = ( + "If a user pastes -----BEGIN PRIVATE KEY----- text, refuse. " + + (words * 700)[: 40 * 1024] + ) + # 10,000 pairs, so a build that looks over the whole held text on every delta shows. + no_space = "Here: " + "a/" * 10_000 + _GH + " done" + saml = ( + "Sign in at https://idp.example.com/sso?SAMLRequest=" + + ("fZJNT8MwDIbvSPyHKPeuWwdbDEFAKE" * 160)[:4200] + + "&tok=" + + _GH + + "&RelayState=Sunshine then come back" + ) + quoted = 'cfg "password": "' + "a b c " * 500 + '" ok' + named_lines = ("token: Zq8wYv3Lp9 and password = abc\n" * 1200)[: 40 * 1024] + + times = { + "honest BEGIN + 40 KiB, 4-char deltas": _seconds( + [refuse[i : i + 4] for i in range(0, len(refuse), 4)] + ), + "20,000 with no space + ghp_, 1-char deltas": _seconds(list(no_space)), + "4 KB SAML query past N, 1-char deltas": _seconds(list(saml)), + "3 KB quoted value, 4-char deltas": _seconds( + [quoted[i : i + 4] for i in range(0, len(quoted), 4)] + ), + "a name on every line, 40 KiB, 4-char deltas": _seconds( + [named_lines[i : i + 4] for i in range(0, len(named_lines), 4)] + ), + } + + assert {name: seconds for name, seconds in times.items() if seconds >= 0.5} == {} + + +def test_a_lone_webhook_broken_by_a_line_break_before_prose_is_cleaned_as_one_url() -> ( + None +): + """C4 (r4 M2): before the text is known to be prose, the seam never cuts at \\n.""" + from fi.alk.providers.agent import _redact + + text = "https://hooks.slack.com/services/" + "T00FAKE00/B00FAKE00/\nFAKEfakeFAKEfakeFAKEfake" + for size in (1, 3, 7): + joined = "".join( + _events_of([text[i : i + size] for i in range(0, len(text), size)]) + ) + assert joined == _redact(text) == "https://hooks.slack.com/services/[redacted]" + + +@pytest.mark.parametrize("at", [4058, 4070, 4080, 4090, 4095]) +def test_a_key_straddling_the_soft_bound_in_a_held_url_query_never_streams( + at: int, +) -> None: + """C4 (r4 M1, r1 L6), at 1-character deltas: no head of the key in any event, and the honest + word after the URL ("then") is kept.""" + from fi.alk.providers.agent import _redact + + url = "https://idp.example.com/sso?SAMLRequest=" + filler = ("fZJNT8MwDIbvSPyHKPeuWwdbDEFAKE" * 160)[: at - len(url) - len("&tok=")] + text = ( + "Sign in at " + + url + + filler + + "&tok=" + + _GH + + "&RelayState=Sunshine then come back" + ) + + events = _events_of(list(text)) + + assert "".join(events) == _redact(text) + assert "".join(events).endswith(" then come back") + assert not any(_GH[:6] in event for event in events) + + +def test_a_key_block_longer_than_the_hard_bound_is_held_never_streamed() -> None: + """C4 (r4 L7): an 18 KB block is held whole, whatever M is.""" + from fi.alk.providers.agent import _redact + + lines = [f"L{i:03d}FAKEQUJDREVGR0hJSktM" + "A" * 36 for i in range(300)] + text = ( + "Key:\n-----BEGIN RSA PRIVATE KEY-----\n" + + "\n".join(lines) + + "\n-----END RSA PRIVATE KEY-----\nKeep it safe." + ) + assert len(text) > 16_384 + + events = _events_of([text[i : i + 7] for i in range(0, len(text), 7)]) + + assert "".join(events) == _redact(text) == "Key:\n[redacted]\nKeep it safe." + assert not any(line[4:30] in event for line in lines for event in events) + + +# -- TH-9500 change review r1 (cr1): what streams past the soft bound --------------------------- + +# cr1 M3: a named value longer than N, plain, quoted with spaces, AWS's guard name, and a URL as +# the value. Past N the shed sent the value cleaned only as far as it was held, then the rest +# with no name in front of it, or the URL whole. +_LONG_NAMED = { + "plain": "Note token: " + "Zq8w" * 1100 + " ok then", + "quoted with spaces": 'cfg "password": "' + "a1 b2 c3 " * 540 + '" ok then', + "aws guard": "aws_secret_access_key = " + + "wJalrXUtnFEMIFAKEK7MD" * 210 + + " ok then", + "a URL as the value": "Note token: https://h.example/" + "a" * 4200 + " ok then", +} + + +@pytest.mark.parametrize("name", list(_LONG_NAMED)) +def test_a_named_value_longer_than_the_soft_bound_streams_as_one_whole_clean( + name: str, +) -> None: + """cr1 M3: held as a key is, up to the hard bound, at 1-, 5- and 50-character deltas.""" + text = _LONG_NAMED[name] + splits = [ + [text[at : at + size] for at in range(0, len(text), size)] + for size in (1, 5, 50) + ] + + assert _split_problems(text, splits) == [] + + +@pytest.mark.parametrize("at", [4070, 4090]) +@pytest.mark.parametrize("size", [1, 5]) +def test_a_key_straddling_the_soft_bound_in_a_run_with_no_space_never_streams( + at: int, size: int +) -> None: + """cr1 M5 (round 3's w3_flush shape): a ghp_ key in a whitespace-free run, across the + 4,096th held character. The key is held whole, up to the hard bound.""" + from fi.alk.providers.agent import _redact + + text = "Here: " + ("a/" * 3000)[: at - 6] + _GH + "/" + "b/" * 300 + " done now" + assert text.find(_GH) == at + + events = _events_of([text[i : i + size] for i in range(0, len(text), size)]) + + assert "".join(events) == _redact(text) + assert not any(_GH[:12] in event or _GH[-20:] in event for event in events) + + +def test_the_seam_stays_linear_on_an_honest_begin_then_256_kib_of_prose() -> None: + """cr1 M6: the per-delta END search is bounded by the delta (r4 H2), at 256 KiB, best of + 3, under 1.5 s. Searched from the block's header on every delta, it took 3.5 s.""" + words = "The quick brown fox jumps over the lazy dog while the build runs. " + refuse = ( + "If a user pastes -----BEGIN PRIVATE KEY----- text, refuse. " + + (words * 4100)[: 256 * 1024] + ) + + seconds = _seconds([refuse[i : i + 4] for i in range(0, len(refuse), 4)]) + + assert seconds < 1.5, f"{seconds:.3f} s" + + +def test_text_after_an_end_split_across_two_deltas_streams_before_the_flush() -> None: + """cr1 L3: an END line split between two deltas closes the block when its second half + arrives, so what follows streams then, not only at the message's end.""" + from fi.alk.harness.hosted_chat_entrypoint import _DeltaCleaner + + text = f"key:\n{_PEM}\nThen restart the service and check the logs." + split = text.index("-----END") + len("-----END RSA PRI") + after = text.index("Then") + cleaner = _DeltaCleaner() + + streamed = "".join( + cleaner.feed(delta) for delta in (text[:split], text[split:after], text[after:]) + ) + + assert streamed.startswith("key:\n[redacted]\nThen restart the service and") + assert ( + streamed + cleaner.flush() + == "key:\n[redacted]\nThen restart the service and check the logs." + ) + + +# -- TH-9500 rule 1b: an @ before the password's / (design addendum v2.5.2) ---------------------- +# The rows test_secret_shapes.py's R1B_CLEANED and R1B_KEPT check on the whole text, copied as +# data. 1b's span and its host lookahead cross no whitespace and no ://, so ChatHold is +# unchanged (L5); a one-URL value runs step 1 on its joined token, as `_clean` does (M3, M-A). +_R1B_ROWS = [ + "postgres://admin:Xk9/Qp+Zr8@db.example.com:5432/app", + "https://user:pa#ss@host.example/x", + "postgres://admin:P@ss/Xk9@db.example.com:5432/app", + "smtp://me@gmail.com:app/pass@smtp.gmail.com:587", + "redis://:p@w/x9@cache.internal:6379/0", + "postgres://admin:P@ss:99/Xk9@db.example.com/app", + "postgres://admin:P@ss/Xk9@localhost/app", + "postgres://admin:P@ss/Xk9@db/app", + "postgres://admin:P@ss/Xk9@[::1]:5432/app", + "postgres://admin:P@ss/Xk9@postgres_db:5432/app", + "postgres://admin:P@ss/Xk9@my_db/app", + "`redis://:p@w/x9@redis`", + "postgres://admin:P@ss/Xk9@db:${DB_PORT}/app", + "postgres://admin:P@ss/Xk9@db:$DB_PORT/app", + "postgres://admin:P@ss/Xk9@db?sslmode=disable", + "redis://:p@w/x9@redis#0", + "(amqp://u:p@w/x9@rabbitmq)", + "amqp://u:p@w/x9@rabbitmq, then", + "amqp://u:p@w/x9@rabbitmq; then", + "", + "[amqp://u:p@w/x9@rabbitmq]", + "postgres://admin:P@ss/Xk9@[fe80::1%25en0]:5432/app", + "a://u:x://v:p@h/x@host.example", + "http://admin@localhost:8080/bob@example.com", + "ssh://git@gitlab.example.com:2222/ops@infra.example/repo.git", + "https://ci@registry.example.com:8443/lodash@4.17.21", + "ssh://git@host.example:22/org@team/repo", + "http://admin@[::1]/bob@example.com", + "ssh://git@[fd00::5]/org@team/repo", + "https://ci@registry.example.com/lodash@4.17.21", + "ssh://git@github.com/org@team/repo", + "Mail https://u:p@h.example/to@~team now", +] +# M3 and M-A: equal and clean when the URL stands alone (in prose, a break inside the span is a +# named limit). +_R1B_BROKEN = { + "postgres://admin:P@ss/Xk\n9@db.example.com:5432/app": "postgres://db.example.com:5432/app", + "postgres://admin:Pa\nss@w/x9@db.example.com/app": "postgres://db.example.com/app", + "redis://:pa\nss@w/x9@cache.internal:6379/0": "redis://cache.internal:6379/0", + "a://u:x://v:p@h/\nx@host.example": "a://host.example", +} + + +@pytest.mark.parametrize("text", _R1B_ROWS, ids=range(len(_R1B_ROWS))) +def test_rule_1b_streams_as_the_whole_text_cleans_at_every_split(text: str) -> None: + """Alone and in prose, every two-way split and char by char, with C1's three checks.""" + for each in (text, f"Use {text} now, then {text} again."): + assert _split_problems(each, _every_split(each))[:3] == [], each + + +@pytest.mark.parametrize("text", list(_R1B_BROKEN), ids=range(len(_R1B_BROKEN))) +def test_a_one_url_value_broken_inside_1bs_span_streams_equal_and_clean( + text: str, +) -> None: + from fi.alk.providers.agent import _redact + + assert _redact(text) == _R1B_BROKEN[text] + assert _split_problems(text, _every_split(text)) == [] + + +# -- TH-9500 change review r2 (cr2) ----------------------------------------------------------------- + +# cr2 H1: a reply that is one URL, with a key or a named value at the start of a later line. +# Cleaning the whole text masks it before the join; before prose, so does the stream. Each row +# is (reply, the secret that must not stream). Owl's ten shapes, then rows longer than N, which +# go through `_shed`'s branch before prose. +_AKIA = "AKIA" + "FAKEFAKEFAKE1234" +_JWT_LONG = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJmYWtlIn0.FAKEsigFAKEsigFAKEsig" +_LONG_URL = "https://h.example/" + "a" * 4100 +_LINE_STARTS = { + "url \\n ghp": ("https://github.com/settings/tokens\n" + _GH, _GH), + "url \\t ghp": ("https://github.com/settings/tokens\t" + _GH, _GH), + "url \\r\\n ghp": ("https://h.example/p\r\n" + _GH, _GH), + "url \\n AKIA": ("https://console.aws.amazon.com/iam\n" + _AKIA, _AKIA), + "url \\n jwt": ("https://jwt.io\n" + _JWT_LONG, "FAKEsigFAKEsigFAKEsig"), + "url \\n sk-proj": ("https://platform.openai.com/api-keys\n" + _SKP, _SKP), + "url \\n token=": ("https://h.example/p\ntoken=Zq8wYv3Lp9Kd2xT", "Zq8wYv3Lp9Kd2xT"), + "url \\n DB_PASSWORD=": ( + "https://h.example/p\nDB_PASSWORD=Zq8wYv3Lp9Kd2xT", + "Zq8wYv3Lp9Kd2xT", + ), + "url \\n url \\n ghp": ("https://a.example/x\nhttps://b.example/y\n" + _GH, _GH), + "1b url \\n ghp": ( + "postgres://admin:P@ss/Xk9@db.example.com:5432/app\n" + _GH, + _GH, + ), + "past N \\n ghp": (_LONG_URL + "\n" + _GH, _GH), + "past N \\n token: \\n value": ( + _LONG_URL + "\ntoken:\nZq8wYv3Lp9Kd2xT", + "Zq8wYv3Lp9Kd2xT", + ), + "past N \\n ghp \\n url": (_LONG_URL + "\n" + _GH + "\nhttps://c.example/z", _GH), + # A URL on the key's own line: the cut there sends the key's line, masked at its start. + "past N \\n ghp/url": (_LONG_URL + "\n" + _GH + "/https://c.example/z", _GH), + "past N, many lines": ("https://h.example/x" + "\nabc" * 1100 + "\n" + _GH, _GH), + # cr3 M5: a key on the line after a slash, glued to a URL. + "past N /\\n AKIA glued to url": ( + _LONG_URL + "/\n" + _AKIA + "https://c.example/z", + _AKIA, + ), +} + + +@pytest.mark.parametrize("name", list(_LINE_STARTS)) +def test_a_key_at_a_line_start_in_a_one_url_reply_never_streams(name: str) -> None: + """cr2 H1: at 1- and 7-character deltas and as one delta, the stream equals cleaning the + whole text, and no event holds the secret's tail, in any case: a value cleaned as part of + a scheme comes out lower-cased (cr3 H1).""" + from fi.alk.providers.agent import _redact + + text, secret = _LINE_STARTS[name] + tail = secret[-12:].lower() + whole = _redact(text) + assert tail not in whole.lower() + for size in (1, 7, len(text)): + events = _events_of([text[i : i + size] for i in range(0, len(text), size)]) + assert "".join(events) == whole, size + assert not any(tail in event.lower() for event in events), size + + +# cr3 H1: past N before prose, a named value or a key on the line before a URL line. The +# branches find URL starts on the joined text, where that line is glued onto the URL's scheme; +# a cut there sent the name alone and cleaned the value as a scheme, lower-cased. Hawk's rows +# 1, 3 and 5. +_V = "Zq8wYv3Lp9Kd2xT" +_GRAFANA = "\n".join( + f"https://grafana.example.com/d/service-{i}/latency" for i in range(90) +) +_LINE_BEFORE_A_URL = { + "a long URL, DB_PASSWORD=V, a URL": ( + _LONG_URL + "\nDB_PASSWORD=" + _V + "\nhttps://b.example/y", + _V, + ), + "a long URL, token=V glued to a URL": ( + _LONG_URL + "\ntoken=" + _V + "https://c.example/z", + _V, + ), + "90 grafana URLs, GITHUB_TOKEN=ghp_, a URL": ( + _GRAFANA + "\nGITHUB_TOKEN=" + _GH + "\nhttps://github.com/org/repo", + _GH, + ), + # A name and its value on the next line go together. + "a long URL, token:, V, a URL": ( + _LONG_URL + "\ntoken:\n" + _V + "\nhttps://c.example/z", + _V, + ), + # A cut never splits a URL's userinfo from its host. + "a long user, :V@host on the next line, a URL": ( + "https://" + "u" * 4100 + "\n:" + _V + "@host.example/x\nhttps://c.example/z", + _V, + ), +} + + +@pytest.mark.parametrize("name", list(_LINE_BEFORE_A_URL)) +@pytest.mark.parametrize("size", [1, 4, 512, 0]) +def test_a_value_on_the_line_before_a_url_never_streams(name: str, size: int) -> None: + """cr3 H1: before prose, a shed cuts only at a line boundary, at 1-, 4- and 512-character + deltas and as one delta.""" + from fi.alk.providers.agent import _redact + + text, secret = _LINE_BEFORE_A_URL[name] + size = size or len(text) + tail = secret[-12:].lower() + whole = _redact(text) + assert tail not in whole.lower() + + events = _events_of([text[i : i + size] for i in range(0, len(text), size)]) + + assert "".join(events) == whole + assert not any(tail in event.lower() for event in events) + + +@pytest.mark.parametrize("size", [1, 7, 0]) +def test_a_line_start_masked_in_the_stream_is_logged_once( + size: int, caplog: pytest.LogCaptureFixture +) -> None: + """cr2 H1: the line-start mask's kinds go on the cleaner's one log line.""" + text = "https://github.com/settings/tokens\n" + _GH + size = size or len(text) + with caplog.at_level("WARNING"): + _events_of([text[i : i + size] for i in range(0, len(text), size)]) + + assert [ + r.getMessage() for r in caplog.records if r.name == "fi.alk.providers.agent" + ] == ["secrets redacted: kinds=github"] + + +@pytest.mark.parametrize("size", [1, 7, 0]) +def test_a_line_start_masked_by_a_shed_is_logged_once( + size: int, caplog: pytest.LogCaptureFixture +) -> None: + """cr3 L3: past N, the shed sends the key's line, masked, and logs its kind once.""" + text = _LONG_URL + "\n" + _GH + "/\nhttps://c.example/z" + size = size or len(text) + with caplog.at_level("WARNING"): + events = _events_of([text[i : i + size] for i in range(0, len(text), size)]) + + assert "[redacted]/" in events[0] + assert [ + r.getMessage() for r in caplog.records if r.name == "fi.alk.providers.agent" + ] == ["secrets redacted: kinds=github"] + + +# cr3 M6 and L1: an opening that fills N with scheme characters and no :// is prose, so the +# URL lines after it aren't glued onto it. L1's named limit ("an opening of 4,096 or more scheme +# characters with no ://") is closed by it, and its row now passes. +_LONG_OPENINGS = { + "M6: a query value on a later URL line": ( + "Zq8w" * 1100 + "\nhttps://registry.example.com/x\n&tok=" + _V, + _V, + ), + "L1: a key on the next line": ("a" * 4100 + "\n" + _GH, _GH), +} + + +@pytest.mark.parametrize("name", list(_LONG_OPENINGS)) +@pytest.mark.parametrize("size", [1, 7, 64, 0]) +def test_a_long_opening_with_no_scheme_streams_as_prose(name: str, size: int) -> None: + from fi.alk.providers.agent import _redact + + text, secret = _LONG_OPENINGS[name] + size = size or len(text) + tail = secret[-12:].lower() + whole = _redact(text) + assert tail not in whole.lower() + + events = _events_of([text[i : i + size] for i in range(0, len(text), size)]) + + assert "".join(events) == whole + assert not any(tail in event.lower() for event in events) + + +# cr2 M2: rule 1 across whitespace, then rule 2's next piece (S's r1r2.py). +_RULE1_THEN_RULE2 = [ + "See redis://:pa ss@h:6379\n?code=s3cr3tvalue99 then", + "See redis://u:pa ss@h.example:6379 &tok=s3cr3tvalue99 then", + "Try postgres://admin:Xk9/Qp+Zr\n 8@db.example.com:5432/app\n&tok=s3cr3tvalue99 ok", + # cr3 M2: the address piece's query ends in &, so rule 2 reaches the next piece. + "See redis://:pa ss@h:6379/?a=1& tok=s3cr3tvalue99 then", +] + + +@pytest.mark.parametrize("text", _RULE1_THEN_RULE2, ids=range(len(_RULE1_THEN_RULE2))) +def test_rule_2s_next_piece_holds_after_a_password_broken_by_whitespace( + text: str, +) -> None: + """cr2 M2: the piece a rule 1 match ends in is an address piece, at 1- and 5-character + deltas, and at every two-way split, where the @ arrives in the delta its piece ends in + (cr3 M2).""" + splits = [ + [text[i : i + size] for i in range(0, len(text), size)] for size in (1, 5) + ] + _every_split(text) + + assert _split_problems(text, splits) == [] + assert not any("s3cr3tvalue99" in event for event in _events_of(list(text))) + + +def test_a_password_broken_by_a_crlf_split_between_deltas_never_streams() -> None: + """cr2 M3: `\\r` and `\\n` in separate deltas, at 1-, 2- and 3-character deltas; and with + spaces and a tab before the `\\r` (cr3 M3).""" + for text in ( + "Try postgres://admin:Xk9/Qp+Zr\r\n8@db.example.com:5432/app now", + "Try postgres://admin:Xk9/Qp+Zr \t\r\n8@db.example.com:5432/app now", + ): + splits = [ + [text[i : i + size] for i in range(0, len(text), size)] + for size in (1, 2, 3) + ] + + assert _split_problems(text, splits) == [], text + + +def test_an_open_quoted_value_longer_than_the_hard_bound_still_streams() -> None: + """cr2 M4: a named value is held up to M (`_LONGEST_HELD_KEY`), not past it: text streams + before the message ends.""" + from fi.alk.harness.hosted_chat_entrypoint import _DeltaCleaner + + words = "The quick brown fox jumps over the lazy dog while the build runs. " + text = 'Set "password": "' + (words * 320)[: 20 * 1024] + cleaner = _DeltaCleaner() + + streamed = "".join(cleaner.feed(text[i : i + 4]) for i in range(0, len(text), 4)) + + assert len(streamed) > 1024 + + +def test_one_large_delta_of_passwords_broken_by_spaces_stays_linear() -> None: + """cr3 M1: `_address` finds its piece by bisection. One 256 KiB delta of rule 1 matches + that cross a space takes at most twice the same text with no broken password, best of 3, + interleaved (cr4 L1: an absolute time failed on a loaded machine).""" + import time + + from fi.alk.harness.hosted_chat_entrypoint import _DeltaCleaner + + def text(unit: str) -> str: + return (unit * (256 * 1024 // len(unit) + 1))[: 256 * 1024] + + broken, control = ( + text("See redis://:pa ss@h x y z "), + text("See redis://h:6379 x y z now "), + ) + best = {broken: 99.0, control: 99.0} + for _ in range(3): + for each in (broken, control): + cleaner = _DeltaCleaner() + start = time.perf_counter() + cleaner.feed(each) + best[each] = min(best[each], time.perf_counter() - start) + + assert best[broken] <= 2 * best[control], ( + f"{best[broken]:.3f} s, {best[control]:.3f} s" + ) + + +# -- TH-9500 change review r4 (cr4) ----------------------------------------------------------------- + +# cr4 H1: Dileep's rule 1b input with its password broken by a line break, after the / and +# before it, past N before prose. A cut between the password's lines sent its first half; the +# cut holds while the text after it, up to the next ://, may still be that URL's userinfo. +_RULE1B_BROKEN_PAST_N = { + "after the /": _LONG_URL + + "\npostgres://admin:Xk9/Qp+Zr\n8@db.example.com:5432/app\nhttps://c.example/z", + "before the /": _LONG_URL + + "\npostgres://admin:Xk\n9/Qp+Zr8@db.example.com:5432/app\nhttps://c.example/z", +} + + +@pytest.mark.parametrize("name", list(_RULE1B_BROKEN_PAST_N)) +@pytest.mark.parametrize("size", [1, 7, 512, 0]) +def test_a_rule_1b_password_broken_by_a_line_break_past_n_never_streams( + name: str, size: int +) -> None: + from fi.alk.providers.agent import _redact + + text = _RULE1B_BROKEN_PAST_N[name] + size = size or len(text) + whole = _redact(text) + assert "qp+zr" not in whole.lower() + + events = _events_of([text[i : i + size] for i in range(0, len(text), size)]) + + assert "".join(events) == whole + assert not any("qp+zr" in event.lower() for event in events) + + +# cr4 M1: honest replies that are only URLs, one per line. Each line ends in scheme characters, +# so joined it is glued onto the next line's scheme; the cut must still fall between them. +_URL_LISTS = { + "400 grafana dashboards": "\n".join( + f"https://grafana.example.com/d/service-{i}/latency" for i in range(400) + ), + "800 bare hosts": "\n".join(f"https://host{i}.example.com" for i in range(800)), + "400 GitHub repositories": "\n".join( + f"https://github.com/org-{i}/repo-{i}" for i in range(400) + ), +} + + +@pytest.mark.parametrize("name", list(_URL_LISTS)) +@pytest.mark.parametrize("size", [7, 512, 0]) +def test_an_honest_url_list_streams_before_its_end(name: str, size: int) -> None: + """cr4 M1: the stream equals the whole text, and most of it goes before the flush.""" + from fi.alk.harness.hosted_chat_entrypoint import _DeltaCleaner + from fi.alk.providers.agent import _redact + + text = _URL_LISTS[name] + size = size or len(text) + cleaner = _DeltaCleaner() + + streamed = "".join( + cleaner.feed(text[i : i + size]) for i in range(0, len(text), size) + ) + + assert streamed + cleaner.flush() == _redact(text) + assert len(streamed) > len(text) // 2 + + +def _best_interleaved(texts: dict[str, str], size: int = 4) -> dict[str, float]: + """Each text fed in `size`-character deltas, best of 3, the texts taking turns so that a + loaded machine slows them alike.""" + import time + + from fi.alk.harness.hosted_chat_entrypoint import _DeltaCleaner + + best = dict.fromkeys(texts, 99.0) + for _ in range(3): + for name, text in texts.items(): + cleaner = _DeltaCleaner() + start = time.perf_counter() + for i in range(0, len(text), size): + cleaner.feed(text[i : i + size]) + cleaner.flush() + best[name] = min(best[name], time.perf_counter() - start) + return best + + +# 16 KiB held in 4-character deltas costs the same for any text it holds, a cut that is walked +# once or one that is walked again on every delta aside; one URL with no breaks is the control. +_HELD_16K = 16 * 1024 +_HELD_CONTROL = ("https://a.example/" + "x" * _HELD_16K)[:_HELD_16K] + + +def test_a_held_cut_is_not_walked_again_on_every_delta() -> None: + """cr4 M2: while a cut holds, the held text grows by lines it can't cut at; they are walked + once, not on every delta. R's shape at 16 KiB in 4-character deltas takes at most twice a + control (walked on every delta, about a minute).""" + held = ("https://hostA\nhttps://hostB" + "\nx" * _HELD_16K)[:_HELD_16K] + # The cut held at a line that is only a URL's own `://` keeps its place too. + at_scheme = ("https://hostA/a\nhttps\n://hostB/p\nhttps://c/" + "\nx" * _HELD_16K)[:_HELD_16K] + + best = _best_interleaved({"held": held, "at_scheme": at_scheme, "control": _HELD_CONTROL}) + + assert best["held"] <= 2 * best["control"], best + assert best["at_scheme"] <= 2 * best["control"], best + + +def test_a_cut_the_userinfo_guard_holds_is_not_walked_again_on_every_delta() -> None: + """cr4b M2: the other way a cut is held: an @ line before the next URL. The guard path + keeps the held cut too.""" + held = ( + "https://a.example/\nalice@example.com\nhttps://b.example/" + "\nx" * _HELD_16K + )[:_HELD_16K] + + best = _best_interleaved({"held": held, "control": _HELD_CONTROL}) + + assert best["held"] <= 2 * best["control"], best + + +@pytest.mark.parametrize("size", [7, 512, 0]) +def test_a_cut_held_by_an_at_line_is_released_by_the_url_lines_after_it( + size: int, +) -> None: + """cr4b M2: a long URL, an email line, then 800 URL lines (24,844 joined). The cut the + @ line holds moves on with the URL starts after it: the list streams as stored, not as + `[redacted]`, at every size (a held cut that stays put past a later start fails at 7).""" + from fi.alk.providers.agent import _redact + + text = ( + _LONG_URL + + "\nalice@example.com\nhttps://c.example/z" + + "".join(f"\nhttps://h{i}.example.com/p" for i in range(800)) + ) + step = size or len(text) + + events = _events_of([text[i : i + step] for i in range(0, len(text), step)]) + + assert "".join(events).strip() == _redact(text.strip()) + assert sum(len(event) for event in events[:-1]) > len(text) // 2 + + +def test_tab_split_lines_are_cut_without_one_call_blocking() -> None: + """cr4b H3: a URL whose lines are split by tabs, 1 MiB of them, then a :// that moves the + cut. No call takes a second (a search back to the held text's start for every line took + 2 s at 1 MiB, 9 s at 2), and the held text, breaks counted, stays within the hard bound.""" + import time + + from fi.alk.harness.hosted_chat_entrypoint import _HOLD_HARD, _DeltaCleaner + + run = 128 + text = ( + "https://a.example/@x" + + ("\t" * run + "x") * (1024 * 1024 // (run + 1)) + + "://b" + ) + cleaner = _DeltaCleaner() + slowest = most = 0 + for i in range(0, len(text), 4096): + start = time.perf_counter() + cleaner.feed(text[i : i + 4096]) + slowest = max(slowest, time.perf_counter() - start) + most = max(most, len(cleaner.held)) + cleaner.flush() + + assert slowest < 1.0, f"{slowest:.2f} s" + assert most <= _HOLD_HARD + 4096 + + +def test_the_walk_over_tab_split_lines_costs_each_line_its_own_length() -> None: + """cr4b H3: `_line_cut` on 1 MiB of tab-split lines, with the cut on the first, takes at + most twice the same on newline-split lines (searching back for every line took 2 s).""" + import time + + from fi.alk.harness.hosted_chat_entrypoint import _URL_BREAKS, _DeltaCleaner + + run = 128 + unit = ("\t" * run + "x", "\n" * run + "x") + best = [99.0, 99.0] + for _ in range(3): + for n, each in enumerate(unit): + raw = "https://a.example/@x" + each * (1024 * 1024 // (run + 1)) + "://b" + start = time.perf_counter() + _DeltaCleaner()._line_cut(raw, raw.translate(_URL_BREAKS), 0) + best[n] = min(best[n], time.perf_counter() - start) + + assert best[0] <= 2 * best[1] + 0.05, best + + +def test_a_name_its_equals_and_its_value_on_three_lines_past_n_is_a_named_limit() -> ( + None +): + """cr4 M3, a named limit: before prose and past N, a name, its `=` and its value each on a + line of its own, then a URL line. No cut keeps all three lines with the name: the value + streams joined onto the URL's scheme, lower-cased, where the whole text masks it. The + stream is pinned as it is, at every size.""" + text = _LONG_URL + "\ntoken\t=\tZq8wYv3Lp9Kd2xT\nhttps://c.example/z" + for size in (1, 7, 512, len(text)): + events = _events_of([text[i : i + size] for i in range(0, len(text), size)]) + assert "".join(events) == ( + _LONG_URL + "token=zq8wyv3lp9kd2xthttps://c.example/z" + ), size + + +# -- TH-9500 change review r5 (cr5): the offset sweeps as tests ------------------------------------- + +# Three rounds of hand-picked rows each passed and then failed the next offset class a sweep +# found. So each class is swept here: a break at every offset after the ://, or at every place +# the input has one, and the rule is the one chat has: the stream, joined, is the stored message +# (`_redact(reply.strip())`), and no event holds the secret, compared lower-cased. A sweep that +# ran fewer cases than it should have, or whose check was taken out, fails on its count. +_SWEEP_BREAKS = {"n": "\n", "t": "\t", "rn": "\r\n"} +_SWEEP_GRAFANA = "\n".join( + f"https://grafana.example.com/d/service-{i}/latency" for i in range(90) +) +# (url, the tail of the password that must never stream) +_SWEEP_URLS = { + "1b": ("postgres://admin:Xk9/Qp+Zr8@db.example.com:5432/app", "qp+zr"), + "r1": ("postgres://admin:S3cretPw9x@db.example.com/app", "cretp"), + "r1-nouser": ("redis://:S3cretPw9x@cache.internal:6379/0", "cretp"), + "1b-at": ("postgres://admin:P@ss/Xk9@db.example.com:5432/app", "ss/xk"), + "1b-ash": ("redis://:p@w/x9@cache.internal:6379/0", "w/x9"), + "1b-pass": ("postgres://admin:Pass@w/x9@db.example.com/app", "w/x9"), +} +_SWEEP_TAILS = ("\nhttps://c.example/z", "https://c.example/z") + + +def _swept(cases, size: int) -> tuple[int, list[str], int]: + """(cases held, the ones that were not, cases skipped) over `cases` of (label, text, + secret): each streamed in deltas of `size`. A case whose stored message keeps the secret is + skipped: that is the base cleaner's, not the stream's.""" + from fi.alk.providers.agent import _redact + + held, bad, skipped = 0, [], 0 + for label, text, secret in cases: + whole = _redact(text.strip()) + if secret in whole.lower(): + skipped += 1 + continue + events = _events_of([text[i : i + size] for i in range(0, len(text), size)]) + joined = "".join(events) + leak = any(secret in event.lower() for event in events) + if leak or joined.strip() != whole: + bad.append(f"{label}: {'leak' if leak else 'unequal'}") + else: + held += 1 + return held, bad, skipped + + +def _assert_swept(cases, size: int, at_least: int) -> None: + held, bad, skipped = _swept(cases, size) + assert not bad, f"{len(bad)} of {held + len(bad)}: {bad[:6]}" + # Every case is held or bad, so an emptied sweep, or a check taken out, changes this. + assert held >= at_least, f"swept {held}, skipped {skipped}" + + +def test_a_sweep_reports_a_leak_and_an_unequal_stream(monkeypatch) -> None: + """The sweeps' own check, on streams made to fail it: a sweep that cannot see a leak or a + difference would pass on any tree.""" + import sys + + here = sys.modules[__name__] + case = [("c", "See https://h.example/p now", "zzzzz")] + + monkeypatch.setattr( + here, "_events_of", lambda deltas: ["See https://h.example/p now"] + ) + assert _swept(case, 5) == (1, [], 0) + monkeypatch.setattr( + here, "_events_of", lambda deltas: ["See https://h.example/p ZZZZZ"] + ) + assert _swept(case, 5) == (0, ["c: leak"], 0) + monkeypatch.setattr( + here, "_events_of", lambda deltas: ["See https://h.example/q now"] + ) + assert _swept(case, 5) == (0, ["c: unequal"], 0) + # The base cleaner's own miss is no stream's: skipped, not held. + assert _swept([("k", "See zzzzz now", "zzzzz")], 5) == (0, [], 1) + + +def _url_sweep(size: int, brk: str): + """A credentialed URL after a long URL, with the break at every offset after its ://.""" + # One delta per character over the long URL alone keeps the 1-character run short. + prefixes = (_LONG_URL,) if size == 1 else (_LONG_URL, _SWEEP_GRAFANA) + tails = _SWEEP_TAILS[:1] if size == 1 else _SWEEP_TAILS + for pn, prefix in enumerate(prefixes): + for name, (url, secret) in _SWEEP_URLS.items(): + for k in range(url.index("://") + 3, len(url) + 1): + for tn, tail in enumerate(tails): + text = prefix + "\n" + url[:k] + brk + url[k:] + tail + yield f"{name}/{pn}/{k}/{tn}", text, secret + + +@pytest.mark.parametrize("brk", list(_SWEEP_BREAKS)) +@pytest.mark.parametrize("size", [1, 7, 64, 512]) +def test_a_credentialed_url_broken_at_any_offset_past_n_streams_as_stored( + size: int, brk: str +) -> None: + """cr4b H1, H2: U's sweep. After a long URL (or a URL list), a credentialed URL with a + break at every offset from just after its :// to its end, before and after its @, inside its + host, and before its port.""" + _assert_swept(_url_sweep(size, _SWEEP_BREAKS[brk]), size, 215 if size == 1 else 860) + + +def _rule_1b_sweep(brk: str): + """R's: a rule 1b password (it holds a /) broken after each of its characters.""" + for pn, prefix in enumerate((_LONG_URL, _SWEEP_GRAFANA)): + for password in ("Xk9/Qp+Zr8", "Xk9/Qp+Zr8Wv5Ty", "ab/cdEFgh12"): + for k in range(1, len(password) + 1): + for tn, tail in enumerate( + ("\nhttps://c.example/z", "https://c.example/z") + ): + text = ( + prefix + + "\npostgres://admin:" + + password[:k] + + brk + + password[k:] + + "@db.example.com:5432/app" + + tail + ) + yield ( + f"{pn}/{password}/{k}/{tn}", + text, + password.split("/")[1][:5].lower(), + ) + + +@pytest.mark.parametrize("brk", list(_SWEEP_BREAKS)) +@pytest.mark.parametrize("size", [1, 7, 64, 512]) +def test_a_rule_1b_password_broken_at_any_offset_past_n_streams_as_stored( + size: int, brk: str +) -> None: + """cr4 H1: R's sweep, a break after each character of a password that holds a /.""" + _assert_swept(_rule_1b_sweep(_SWEEP_BREAKS[brk]), size, 144) + + +def _dodge_cases(brk: str): + """Every :// held is broken by a line break, so the guard that looks for one finds none + (cr4b H2), and rule 1b passwords broken after the @ or inside the host (cr4b H1).""" + long = _LONG_URL + "/" + z = "https://c.example/" + "z" * 4100 + cases = { + "a :/ break /": ( + long + "\npostgres:/\n/admin:Xk9/Qp+Zr\n8@db.example.com:5432/app\n" + z, + "qp+zr", + ), + "b : break //": ( + long + "\npostgres:\n//admin:Xk9/Qp+Zr\n8@db.example.com:5432/app\n" + z, + "qp+zr", + ), + "c :/ break /, P@ss/Xk9@": ( + long + "\npostgres:/\n/admin:P@ss/Xk9@\ndb.example.com:5432/app\n" + z, + "ss/xk9", + ), + "d pw broken twice": ( + long + "\npostgres:/\n/admin:Xk9/Qp\n+Zr\n8@db.example.com:5432/app\n" + z, + "qp+zr", + ), + "e control, :// whole": ( + long + "\npostgres://admin:Xk9/Qp+Zr\n8@db.example.com:5432/app\n" + z, + "qp+zr", + ), + } + long_url = _LONG_URL + other = "https://c.example/z" + h123 = { + "H1 broken after the path's @": ( + long_url + + "\npostgres://admin:P@ss/Xk9@\ndb.example.com:5432/app\n" + + other, + "ss/xk9", + ), + "H1 addendum shape, port in password": ( + long_url + "\npostgres://admin:P@ss:99/Xk9@\ndb.example.com/app\n" + other, + "ss:99/xk9", + ), + "H1 redis 1b, glued URL after": ( + long_url + "\nredis://:p@w/x9@\ncache.internal:6379/0" + other, + "w/x9", + ), + "H1 broken inside the host": ( + long_url + + "\npostgres://admin:P@ss/Xk9@db.\nexample.com:5432/app\n" + + other, + "ss/xk9", + ), + "H1 control, under N": ( + "https://h.example/p\npostgres://admin:P@ss/Xk9@\ndb.example.com:5432/app\n" + + other, + "ss/xk9", + ), + "H2 :/ break /, password broken": ( + long_url + + "/\npostgres:/\n/admin:Xk9/Qp+Zr\n8@db.example.com:5432/app\nhttps://c.example/" + + "z" * 4100, + "qp+zr", + ), + "H2 control, :// whole": ( + long_url + + "/\npostgres://admin:Xk9/Qp+Zr\n8@db.example.com:5432/app\nhttps://c.example/" + + "z" * 4100, + "qp+zr", + ), + } + for name, (text, secret) in {**cases, **h123}.items(): + yield name, text.replace("\n", brk), secret + + +@pytest.mark.parametrize("brk", list(_SWEEP_BREAKS)) +@pytest.mark.parametrize("size", [1, 7, 64, 512]) +def test_a_url_whose_own_scheme_or_host_is_broken_past_n_streams_as_stored( + size: int, brk: str +) -> None: + """cr4b H1, H2: the guard may not depend on a :// being whole in the text held, nor on the + break falling in a place that leaves the password's host on the next line.""" + _assert_swept(_dodge_cases(_SWEEP_BREAKS[brk]), size, 12) + + +def _honest_lists() -> dict[str, str]: + def alt(n: int, a: str, b: str) -> str: + return "\n".join(a.format(i=i) + "\n" + b.format(i=i) for i in range(n)) + + return { + "github + email x300": alt( + 300, "https://github.com/user{i}", "user{i}@example.com" + ), + "email + github x300": "https://example.com/team\n" + + "\n".join( + f"user{i}@example.com\nhttps://github.com/user{i}" for i in range(300) + ), + "url + git@ x300": alt( + 300, + "https://gitlab.example.com/org/repo{i}", + "git@gitlab.example.com:org/repo{i}.git", + ), + "url + mailto x300": alt( + 300, "https://example.com/people/p{i}", "mailto:p{i}@example.com" + ), + "url + @mention x400": alt( + 400, "https://github.com/org/repo{i}/pull/{i}", "@reviewer{i}" + ), + "url + img@sha256 x150": alt( + 150, + "https://hub.docker.com/r/org/img{i}", + "org/img{i}@sha256:" + "ab12" * 16, + ), + "medium /@user x500": "\n".join( + f"https://medium.com/@writer{i}" for i in range(500) + ), + "npm /@scope/pkg x500": "\n".join( + f"https://www.npmjs.com/package/@scope{i}/pkg" for i in range(500) + ), + "control: github only x600": "\n".join( + f"https://github.com/user{i}" for i in range(600) + ), + } + + +# The lists that stream: an @ in a URL's own path or a /@ segment is no userinfo. +_STREAMING_LISTS = ( + "medium /@user x500", + "npm /@scope/pkg x500", + "control: github only x600", +) +# cr4b M1, a named limit: an @ on the line before a URL could be the end of a userinfo, and the +# cut after it can't tell, so the held text isn't cut while a later URL follows an @ line. Past +# _HOLD_HARD it is sent as `[redacted]` and the rest of the token is dropped; short of it, +# nothing goes before the flush. (Holding only while rule 1 finds a userinfo that would cross +# the cut was tried: rule 1 is the cross-whitespace rule, and the plain userinfo of a URL on one +# line isn't its span, so it let passwords through.) The raw text is counted toward the bound, +# breaks too, so a list of about 16,000 joined characters with a break per URL is past it. +_AT_LINE_LISTS_REDACTED = ( + "url + git@ x300", + "url + @mention x400", + "url + img@sha256 x150", + "url + mailto x300", +) +_AT_LINE_LISTS_HELD = ("github + email x300", "email + github x300") + + +def _fed(text: str, size: int) -> tuple[str, str]: + """What the stream sends before the flush, and what the flush sends.""" + from fi.alk.harness.hosted_chat_entrypoint import _DeltaCleaner + + cleaner = _DeltaCleaner() + sent = "".join(cleaner.feed(text[i : i + size]) for i in range(0, len(text), size)) + return sent, cleaner.flush() + + +@pytest.mark.parametrize("size", [7, 512, 0]) +def test_an_honest_url_list_streams_as_stored_and_before_its_end(size: int) -> None: + """URL-only lists past _HOLD_SOFT joined with an @ in a path: the stream is the stored + message, and most of it goes before the flush.""" + from fi.alk.providers.agent import _redact + + lists = _honest_lists() + for name in _STREAMING_LISTS: + text = lists[name] + sent, flushed = _fed(text, size or len(text)) + assert (sent + flushed).strip() == _redact(text.strip()), name + assert len(sent) > len(text) // 2, name + assert ( + len(_STREAMING_LISTS) + len(_AT_LINE_LISTS_REDACTED) + len(_AT_LINE_LISTS_HELD) + == 9 + ) + assert len(lists) == 9 + + +@pytest.mark.parametrize("size", [7, 512, 0]) +def test_an_honest_url_list_with_an_at_line_before_each_url_is_a_named_limit( + size: int, +) -> None: + """cr4b M1, a named limit: pinned as it is, so that a change to it shows.""" + from fi.alk.providers.agent import _redact + + lists = _honest_lists() + for name in _AT_LINE_LISTS_REDACTED + _AT_LINE_LISTS_HELD: + text = lists[name] + sent, flushed = _fed(text, size or len(text)) + if name in _AT_LINE_LISTS_REDACTED: + assert sent + flushed == "[redacted]", name + else: + assert sent == "", name + assert flushed.strip() == _redact(text.strip()), name + + +# -- TH-9500 change review r5, H1: a break just before a credentialed URL's own :// ----------------- +_PRE_SCHEME_URLS = { + "1b": ("postgres://admin:Xk9/Qp+Zr8@db.example.com:5432/app", "qp+zr"), + "1b-at": ("postgres://admin:P@ss/Xk9@db.example.com:5432/app", "ss/xk"), + "1b-port": ("postgres://admin:P@ss:99/Xk9@db.example.com/app", "ss:99"), + "1b-ash": ("redis://:p@w/x9@cache.internal:6379/0", "w/x9"), + "r1": ("postgres://admin:S3cretPw9x@db.example.com/app", "cretp"), +} + + +def _pre_scheme_cases(): + for name, (url, secret) in _PRE_SCHEME_URLS.items(): + at = url.index("://") + for kind, brk in _SWEEP_BREAKS.items(): + text = _LONG_URL + "\n" + url[:at] + brk + url[at:] + "\nhttps://c.example/z" + yield f"{name}/{kind}", text, secret + + +@pytest.mark.parametrize("size", [1, 7, 64, 512, 0], ids=lambda size: f"size_{size or 'whole'}") +def test_a_credentialed_url_broken_just_before_its_scheme_past_n_streams_as_stored(size) -> None: + """Past N before prose, the line cut can fall between a URL's scheme and its own ://, where + the text after the cut still holds the userinfo. Each of 5 URLs × 3 breaks streams equal to + the stored message, with no part of its password in any event.""" + cases = list(_pre_scheme_cases()) + assert len(cases) == 15 + if size == 0: + held = sum(_swept([case], len(case[1]))[0] for case in cases) + bad = [b for case in cases for b in _swept([case], len(case[1]))[1]] + assert not bad, bad + assert held == len(cases) + return + _assert_swept(cases, size, at_least=len(cases)) + + +@pytest.mark.parametrize("size", [1, 7, 64]) +def test_a_scheme_line_then_a_lone_colon_line_past_n_streams_as_stored(size) -> None: + """The scheme, its `:` and its `//` on three lines, with a password holding `://` and an @ + on a line of its own: the cut must not fall between the scheme and its `://`.""" + text = ( + _LONG_URL + + "\npostgres\n:\n//admin:S3cretPw9x/a://b\n@\ndb.example.com/app\nhttps://c.example/z" + ) + _assert_swept([("scheme, :, //", text, "s3cretpw9x")], size, at_least=1) diff --git a/tests/harness/test_contract_writer.py b/tests/harness/test_contract_writer.py index 2bcfa3e9a..9b6da78ac 100644 --- a/tests/harness/test_contract_writer.py +++ b/tests/harness/test_contract_writer.py @@ -494,7 +494,7 @@ def test_one_save_logs_one_line_however_many_urls_are_refused( logged = [record for record in caplog.records if record.name == "fi.alk.providers.agent"] assert [record.getMessage() for record in logged] == [ - "a URL was redacted whole: class=ValueError" + "secrets redacted: url=ValueError" ] assert PASSWORD not in caplog.text assert PASSWORD not in (tmp_path / "contract.json").read_text(encoding="utf-8") diff --git a/tests/harness/test_hosted_entrypoint.py b/tests/harness/test_hosted_entrypoint.py index 7af13a83e..8245df3ae 100644 --- a/tests/harness/test_hosted_entrypoint.py +++ b/tests/harness/test_hosted_entrypoint.py @@ -5495,6 +5495,26 @@ async def reading(_args): assert "hunter2" not in events and "sk-1" not in events +def test_a_tool_result_is_cleaned_before_it_is_cut_at_600_characters(): + """TH-9500 X1 (plan v1.1 section 6, r1 M1): session.py:388 cleans before it cuts, so a key + past character 597 is masked whole, and a query URL before it can't move the cut rules.""" + from fi.alk.harness.backends.base import SessionSpec, ToolReturned + from fi.alk.harness.session import RESULT, Stage, Turn + + key = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8" + # Cut first, 6 of the key's characters would be left before the "...": too few for the + # rule that masks a key cut short. + text = "see https://h.example/x?a=1 then " + "y " * 277 + key + assert text.index(key) + len("ghp_A1b2C3") == 597 + stage = Stage(SessionSpec(system_prompt=""), name="understand") + + [event] = stage._events(ToolReturned(id="c1", text=text), Turn()) + + assert event.kind == RESULT + assert "A1b2C3" not in event.text + assert event.text == "see https://h.example/x then " + "y " * 277 + "[redacted]" + + def test_the_activity_printed_to_stdout_loses_url_credentials(capsys, monkeypatch): """Change review r1, A-H1: the platform keeps the tail of the guest's stdout.""" from fi.alk.harness import cli diff --git a/tests/harness/test_outbound.py b/tests/harness/test_outbound.py index 92294b857..0dd3b0726 100644 --- a/tests/harness/test_outbound.py +++ b/tests/harness/test_outbound.py @@ -3400,7 +3400,7 @@ def test_build_event_record_redacts_a_dsn_in_world_unhealthy_cause() -> None: ) cause = record["payload"]["cause"] assert "s3cr3t" not in cause - assert cause == "connect failed: postgresql://harness:***@localhost:14000/w0" + assert cause == "connect failed: postgresql://localhost:14000/w0" # The digest must match the REDACTED bytes actually spooled, not the original with the secret. assert record["digest"] == event_payload_digest(record["payload"]) @@ -3423,7 +3423,7 @@ def test_build_event_record_redacts_terminal_failure_message() -> None: ) message = record["payload"]["failure"]["message"] assert "hunter2" not in message - assert "postgresql://harness:***@localhost:14000/w0" in message + assert "postgresql://localhost:14000/w0" in message def test_build_result_receipt_redacts_sub_goal_and_evaluation_reasons_and_failure_message() -> ( @@ -3463,11 +3463,9 @@ def test_build_result_receipt_redacts_sub_goal_and_evaluation_reasons_and_failur }, ) assert "pw123" not in str(receipt) - assert ( - receipt["sub_goals"][0]["reason"] == "leaked postgresql://harness:***@host/db" - ) - assert receipt["evaluations"][0]["reason"] == "saw postgresql://harness:***@host/db" - assert receipt["failure"]["message"] == "postgresql://harness:***@host/db" + assert receipt["sub_goals"][0]["reason"] == "leaked postgresql://host/db" + assert receipt["evaluations"][0]["reason"] == "saw postgresql://host/db" + assert receipt["failure"]["message"] == "postgresql://host/db" # The digest must match the redacted bytes. ResultReceiptDraft.model_validate(receipt) # round-trips cleanly, digest agrees @@ -3475,7 +3473,7 @@ def test_build_result_receipt_redacts_sub_goal_and_evaluation_reasons_and_failur def test_redact_outbound_text_scrubs_userinfo_and_extra_secrets() -> None: assert ( redact_outbound_text("postgresql://harness:pw@localhost/db") - == "postgresql://harness:***@localhost/db" + == "postgresql://localhost/db" ) assert redact_outbound_text("no secrets here") == "no secrets here" assert ( @@ -3498,12 +3496,12 @@ def test_redact_outbound_text_scrubs_userinfo_and_extra_secrets() -> None: [ # P3: the empty-username shape -- the canonical Redis/RabbitMQ/Mongo DSN form -- used to # leave the password verbatim (the old pattern required a NON-empty username group). - ("redis://:secretpw@h:6379/0", "redis://:***@h:6379/0"), + ("redis://:secretpw@h:6379/0", "redis://h:6379/0"), # A password containing '@' must not truncate the mask at the first '@' seen. - ("mysql://u:p@ss@h/db", "mysql://u:***@h/db"), + ("mysql://u:p@ss@h/db", "mysql://h/db"), # A bare token/username-only userinfo (no ':') is the standard shape a bearer token takes # in git/registry output -- must be masked outright, not treated as "just a username." - ("https://ghp_TOKEN@github.com/o/r.git", "https://***@github.com/o/r.git"), + ("https://ghp_TOKEN@github.com/o/r.git", "https://github.com/o/r.git"), # Negative: a path segment containing '@' with no userinfo before it must be left alone. ("https://example.com/a@b", "https://example.com/a@b"), # Negative: no "://" at all -- not a userinfo shape. @@ -3516,6 +3514,73 @@ def test_redact_outbound_text_handles_the_empty_username_and_bare_token_userinfo assert redact_outbound_text(raw) == expected +# -- TH-9500: the kit's one cleaner runs here too (plan v1.1 section 4, test O1) ---------------- + +_FAKE_KEY = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8" + + +def test_a_declared_secret_is_matched_whole_before_the_cleaner_can_mask_part_of_it() -> ( + None +): + """The order: extra values first. A declared secret holding a word and a key would lose + only its key to the cleaner, and the rest would no longer match it exactly (r1 M5).""" + declared = f"deploy {_FAKE_KEY}" + + out = redact_outbound_text(f"ran {declared} ok", extra_secret_values=(declared,)) + + assert out == "ran *** ok" + assert "deploy" not in out + + +def test_a_key_and_a_broken_password_are_masked_with_one_log_line( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.DEBUG): + out = redact_outbound_text( + f"push with {_FAKE_KEY} to https://bot:hun ter2@h.example/v1 now" + ) + + assert out == "push with [redacted] to https://h.example/v1 now" + assert [ + r.getMessage() for r in caplog.records if r.name == "fi.alk.providers.agent" + ] == ["secrets redacted: kinds=github,rule1"] + + +def test_nothing_masked_logs_nothing(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.DEBUG): + assert ( + redact_outbound_text("token: required, see docs") + == "token: required, see docs" + ) + + assert [r for r in caplog.records if r.name == "fi.alk.providers.agent"] == [] + + +# Rule 1b (design addendum v2.5.2): Dileep's #278 threads P1 and P3, and Ash's two inputs. +_RULE1B_ROWS = { + "postgres://admin:Xk9/Qp+Zr8@db.example.com:5432/app": ( + "postgres://db.example.com:5432/app" + ), + "https://user:pa#ss@host.example/x": "https://host.example/x", + "postgres://admin:P@ss/Xk9@db.example.com:5432/app": ( + "postgres://db.example.com:5432/app" + ), + "smtp://me@gmail.com:app/pass@smtp.gmail.com:587": "smtp://smtp.gmail.com:587", +} + + +@pytest.mark.parametrize("raw", list(_RULE1B_ROWS), ids=range(len(_RULE1B_ROWS))) +def test_a_password_holding_an_at_sign_or_a_slash_leaves_no_part_behind( + raw: str, +) -> None: + expected = _RULE1B_ROWS[raw] + + assert redact_outbound_text(raw) == expected + assert redact_outbound_text(f"failed on {raw} twice") == ( + f"failed on {expected} twice" + ) + + # --- ArtifactsClient ----------------------------------------------------------------------------- diff --git a/tests/harness/test_reading_entry.py b/tests/harness/test_reading_entry.py index a5a062076..262b70bc9 100644 --- a/tests/harness/test_reading_entry.py +++ b/tests/harness/test_reading_entry.py @@ -550,7 +550,7 @@ def crash(*_args): "retryable": True, } assert failure["message"].startswith( - "RuntimeError: could not fetch https://user:***@host/repo.git " + "RuntimeError: could not fetch https://host/repo.git " ) assert "pw" not in failure["message"] assert MESSAGE_LIMIT == 2000 diff --git a/tests/harness/test_scenario_source_provision_agent.py b/tests/harness/test_scenario_source_provision_agent.py index 03f820a69..352eb8066 100644 --- a/tests/harness/test_scenario_source_provision_agent.py +++ b/tests/harness/test_scenario_source_provision_agent.py @@ -143,7 +143,7 @@ def test_a_url_urlsplit_refuses_is_registered_redacted_and_logged_by_class_only( # Once for the document and once for the prompt. logged = [record for record in caplog.records if record.name == "fi.alk.providers.agent"] assert [record.getMessage() for record in logged] == [ - "a URL was redacted whole: class=ValueError" + "secrets redacted: url=ValueError" ] * 2 assert all(record.exc_info is None for record in logged) assert "hunter2" not in caplog.text @@ -452,3 +452,210 @@ def test_the_document_and_the_reading_hold_the_same_cleaned_prompt(tmp_path: Pat assert payload["agent"]["agent"]["behaviour"][0]["instructions"] == cleaned assert reading["agent"]["behaviour"][0]["instructions"] == cleaned assert "hunter2" not in json.dumps(reading) and "sk-1" not in json.dumps(reading) + + +# -- TH-9500: the persona record is cleaned before it leaves (plan v1.1 section 5, test P1) ------ + +_FAKE_KEY = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8" + + +def test_a_key_in_each_cleaned_persona_field_is_masked_and_the_rest_sent_as_written() -> ( + None +): + body = { + "name": f"Dana {_FAKE_KEY}", + "use_case": f"Refund {_FAKE_KEY}", + "instruction": f"She pastes {_FAKE_KEY} into the chat", + "tests": "The agent refuses password=Zq8wYv3Lp9 politely", + "persona": { + "role": f"admin {_FAKE_KEY}", + "background": f"keeps {_FAKE_KEY} in a note", + "secret_motive": "wants a refund", + "traits": ["calm", f"uses {_FAKE_KEY}"], + }, + } + scenario = SimpleNamespace( + presented=ss._presented(body, scenario_key="refund-ghp_key-s1"), + presented_fields=ss._presented( + body, scenario_key="refund-ghp_key-s1", fields=True + ), + scenario_key="refund-ghp_key-s1", + ) + + payload = ss._provision_payload( + "Run ghp_x", [scenario], ["task_completion"], "", "chat", agent_name="Desk" + ) + + [persona] = payload["personas"] + assert _FAKE_KEY not in json.dumps(payload) + assert "Zq8wYv3Lp9" not in json.dumps(payload) + assert persona == { + "name": "Dana [redacted]", + "scenario_name": "Refund [redacted]", + "role": "admin [redacted]", + "situation": "She pastes [redacted] into the chat", + "outcome": "The agent refuses password=[redacted] politely", + "persona": { + "role": "admin [redacted]", + # The key markers apply to the persona's own keys too. + "background": "keeps [redacted] in a note", + "secret_motive": "[redacted]", + "traits": ["calm", "uses [redacted]"], + }, + "scenario_key": "refund-ghp_key-s1", + } + # As written: the platform matches on scenario_key, and the rest is the kit's own. + assert payload["name"] == "Run ghp_x" + assert payload["agent_name"] == "Desk" + assert payload["chosen_evals"] == ["task_completion"] + assert payload["modality"] == "chat" + # The judge's record is left as it was. + assert scenario.presented["name"] == f"Dana {_FAKE_KEY}" + + +def _sent(body: dict[str, Any], scenario_key: str = "s1") -> tuple[dict, dict]: + """The payload's persona for a scenario built from `body` as `_load_one` builds it, and the + record the judge reads.""" + presented = ss._presented(body, scenario_key=scenario_key) + scenario = SimpleNamespace( + presented=presented, + presented_fields=ss._presented(body, scenario_key=scenario_key, fields=True), + scenario_key=scenario_key, + ) + [persona] = ss._provision_payload("Run", [scenario], (), "", "chat")["personas"] + return persona, presented + + +def _straddling(at: int) -> str: + text = ("Caller says: " + "please help me with the refund " * 10)[:at] + text += " " + _FAKE_KEY + " and then hangs up politely." + assert text.index(_FAKE_KEY) == at + 1 + return text + + +def _masked(text: str) -> str: + at = text.index(_FAKE_KEY) + return text[:at] + "[redacted]" + text[at + len(_FAKE_KEY) :] + + +@pytest.mark.parametrize("at", [222, 244]) +@pytest.mark.parametrize("where", ["persona", "name_falls_back_to_the_scenario"]) +def test_a_key_straddling_the_255_character_cut_of_name_and_role_is_masked( + at: int, where: str +) -> None: + """P1 (cr1 M4, cr2 M1): `persona_of` cuts `name` and `role` at 255 characters, so the record + sent is `persona_of` over cleaned fields, including a name that falls back to the scenario's + own. The record the judge reads stays as written.""" + text = _straddling(at) + if where == "persona": + body = {"persona": {"name": text, "role": text}, "name": "s1"} + else: + body = {"persona": {"role": text}, "name": text} + body.update(instruction=text, tests="ok", use_case="Refund") + + persona, presented = _sent(body) + + for field in ("name", "role"): + assert persona[field] == _masked(text)[:255] + assert _FAKE_KEY[:6] not in json.dumps(persona) + assert presented["name"] == presented["role"] == text[:255] + + +@pytest.mark.parametrize("at", [222, 244]) +@pytest.mark.parametrize("where", ["use_case", "tests"]) +def test_a_key_straddling_the_cut_of_scenario_name_is_masked( + at: int, where: str +) -> None: + """cr2 M1: `display_scenario_name` cuts `use_case`, or `tests` when there is no `use_case` + and no `scenario_key` to make one from, at 255 characters.""" + text = _straddling(at) + body = {"persona": {"name": "Dana"}, "name": "s1", "instruction": "x", where: text} + + persona, presented = _sent(body, scenario_key="") + + assert persona["scenario_name"] == _masked(text)[:255] + assert _FAKE_KEY[:6] not in json.dumps(persona) + assert presented["scenario_name"] == text[:255] + + +def test_a_key_in_a_scenario_name_the_display_rewrites_is_masked() -> None: + """cr2 M1: with no `use_case`, `tests` or `scenario_key`, the shown name is the scenario's + `name` with `-` and `_` made spaces, which takes a key out of its shape (`ghp A1b2…`).""" + body = {"persona": {"name": "Dana"}, "name": f"call about {_FAKE_KEY} please"} + + persona, presented = _sent(body, scenario_key="") + + assert persona["scenario_name"] == "Call about [redacted] please" + assert _FAKE_KEY[4:20] not in json.dumps(persona) + assert presented["scenario_name"] == f"Call about ghp {_FAKE_KEY[4:]} please" + + +@pytest.mark.parametrize( + "body", + [ + {"persona": {"name": "N" * 300, "role": "R" * 300}, "name": "S"}, + {"persona": {"occupation": "O" * 300}, "name": "S" * 300}, + {"persona": {}, "name": ""}, + {"persona": "not a dict", "name": "S"}, + {"persona": {"name": "", "role": ""}, "name": "S"}, + {"name": "refund-for_bob", "tests": "", "use_case": ""}, + {}, + ], + ids=[ + "persona", + "occupation_and_fallback", + "caller", + "persona_not_a_dict", + "empty", + "name_rewritten", + "bare", + ], +) +def test_the_record_sent_is_the_record_shown_when_nothing_is_secret(body) -> None: + """cr2 M1, cr3 L4: cleaning first changes no honest field, so each fallback `persona_of` + takes is the same in the record sent.""" + for scenario_key in ("s1", ""): + persona, presented = _sent(body, scenario_key=scenario_key) + assert persona == {**presented, "scenario_key": scenario_key} + + +def test_a_key_the_shown_name_rewrites_into_shape_is_masked() -> None: + """cr3 M4: `call_AKIA…` isn't a key's shape (a key's own character comes before it), but the + shown name makes `_` a space, `Call AKIA…`. The clean after `persona_of` masks it.""" + body = {"persona": {"name": "Dana"}, "name": "call_AKIAFAKEFAKEFAKE1234 now"} + + persona, _ = _sent(body, scenario_key="") + + assert persona["scenario_name"] == "Call [redacted] now" + assert "AKIAFAKE" not in json.dumps(persona) + + +def test_loading_a_scenario_logs_nothing_and_the_send_logs_one_line( + caplog: pytest.LogCaptureFixture, +) -> None: + """cr3 L4: the clean and its log line are the send's. Loading a scenario cleans nothing, so + a run that loads scenarios and never sends them logs nothing.""" + body = {"persona": {"name": f"Dana {_FAKE_KEY}"}, "name": "s1"} + with caplog.at_level(logging.WARNING): + presented = ss._presented(body, scenario_key="s1") + read = ss._presented(body, scenario_key="s1", fields=True) + loaded = [r for r in caplog.records if r.name == "fi.alk.providers.agent"] + scenario = SimpleNamespace( + presented=presented, presented_fields=read, scenario_key="s1" + ) + ss._provision_payload("Run", [scenario], (), "", "chat") + sent = [r for r in caplog.records if r.name == "fi.alk.providers.agent"] + + assert loaded == [] + assert read["persona"]["name"] == f"Dana {_FAKE_KEY}" + assert [r.getMessage() for r in sent] == ["secrets redacted: kinds=github"] + + +def test_a_compiled_scenario_carries_the_fields_to_send() -> None: + """The one place a scenario is built from its document fills `presented_fields`.""" + import inspect + + assert ( + "presented_fields=_presented(body, scenario_key=scenario_key, fields=True)" + in inspect.getsource(ss._load_one) + ) diff --git a/tests/providers/test_providers_agent_build.py b/tests/providers/test_providers_agent_build.py index ba86233a6..fc69d8614 100644 --- a/tests/providers/test_providers_agent_build.py +++ b/tests/providers/test_providers_agent_build.py @@ -109,7 +109,7 @@ def test_a_refused_url_inside_prose_becomes_redacted_and_the_rest_stays( # One line for the call, however many URLs in it were refused (#271). [logged] = [record for record in caplog.records if record.name == "fi.alk.providers.agent"] - assert logged.getMessage() == "a URL was redacted whole: class=ValueError" + assert logged.getMessage() == "secrets redacted: url=ValueError" assert "hunter2" not in caplog.text @@ -140,11 +140,61 @@ def test_many_refused_urls_in_one_document_log_one_line( assert agent["behaviour"][0]["instructions"] == "[redacted]" assert agent["tools"][0]["parameters"]["properties"]["k"]["enum"] == ["[redacted]"] * 1001 [logged] = [record for record in caplog.records if record.name == "fi.alk.providers.agent"] - assert logged.getMessage() == "a URL was redacted whole: class=ValueError" + assert logged.getMessage() == "secrets redacted: url=ValueError" assert logged.exc_info is None assert "hunter2" not in caplog.text +def test_the_log_line_names_the_url_class_and_the_kinds_once_per_call( + caplog: pytest.LogCaptureFixture, +) -> None: + """TH-9500 (plan v1.1 section 2, S6): one line per call, naming what was removed, never + the text.""" + document = { + "a": "password=Zq8wYv3Lp9", + "b": ["https://u:p@h/x?k=1", "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8"], + "c": "Before https://admin:hunter2\uff03x@db.example/ after", + } + + with caplog.at_level(logging.DEBUG, logger="fi.alk.providers.agent"): + _redact(document) + + [logged] = [ + record for record in caplog.records if record.name == "fi.alk.providers.agent" + ] + assert logged.getMessage() == "secrets redacted: url=ValueError; kinds=github,named" + assert "Zq8wYv3Lp9" not in caplog.text and "hunter2" not in caplog.text + + +def test_the_log_line_names_the_kinds_alone_when_no_url_was_refused( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.DEBUG, logger="fi.alk.providers.agent"): + assert _redact("see https://bot:hun ter2@h/v1") == "see https://h/v1" + + assert [ + r.getMessage() for r in caplog.records if r.name == "fi.alk.providers.agent" + ] == ["secrets redacted: kinds=rule1"] + + +@pytest.mark.parametrize( + "text", + [ + "password: required", + "see https://h.example/x?k=1", + "password=[redacted]", + "no secret", + ], +) +def test_nothing_is_logged_when_no_secret_rule_changed_the_text( + caplog: pytest.LogCaptureFixture, text: str +) -> None: + with caplog.at_level(logging.DEBUG, logger="fi.alk.providers.agent"): + _redact(text) + + assert [r for r in caplog.records if r.name == "fi.alk.providers.agent"] == [] + + # -- TH-9491: a URL inside prose (plan v5.1) ------------------------------------------------------- # No shrinking: each assertion names its input, and a planted bug then fails in seconds. @@ -225,13 +275,104 @@ def _lone_urls(draw: Any) -> str: @_PROPERTY @given(_lone_urls()) def test_a_single_url_field_is_cleaned_as_the_base_cleans_it(url: str) -> None: - """B5: a single-URL field, after W1's join, gives the frozen base's result.""" + """B5: a single-URL field, after W1's join, gives the frozen base's result. TH-9500's rule 1 + first removes a userinfo whose text after "user:" a tab or a line break splits, which is + what the base's join used to read as one password. The oracle runs `leftovers`, so it can't + see a rule 1b condition that fires wrongly or not at all; the rows in + test_secret_shapes.py (R1B_CLEANED, R1B_KEPT, R1B_LIMITS) pin those (cr2 L4).""" from fi.alk.providers.agent import _START + from fi.alk.providers.secret_shapes import leftovers joined = url.replace("\t", "").replace("\r", "").replace("\n", "") if len(_START.findall(joined)) != 1: return - assert _redact(url) == _frozen_base_url(url), url + # A one-URL value runs step 1 on its joined token (v2.5.2, M-A), as chat does. + assert _redact(url) == _frozen_base_url(leftovers(joined)), url + assert _only_userinfo_and_query_values_taken(joined, leftovers(joined)), url + + +_QUERY_VALUE = re.compile(r"[^&#\s]*") + + +def _only_userinfo_and_query_values_taken(url: str, left: str) -> bool: + """What `leftovers` may take from a lone URL, checked without it (cr1 L1): the oracle above + runs `leftovers` itself, so a rule 1 bug moves both sides alike. Rule 1 takes the userinfo + right after the ://, up to and including its one @; rule 2 takes whole query values, each + after an =. Rule 1b takes the userinfo up to the first path segment's @. Nothing else + changes. It bounds what may go, not when: a 1b condition that fires wrongly or not at all + passes it, and the rows in test_secret_shapes.py pin those (cr2 L4).""" + head = url.index("://") + 3 + if left[:head] != url[:head]: + return False + rests = [url[head:]] + at = url.find("@", head) + if at >= 0: + rests.append(url[at + 1 :]) + slash = url.find("/", head) + path_at = url.find("@", slash) if slash >= 0 else -1 + if path_at >= 0: + rests.append(url[path_at + 1 :]) + return any(_values_taken(rest, left[head:]) for rest in rests) + + +def _values_taken(full: str, left: str) -> bool: + """Whether `left` is `full` less whole query values. A value taken is the whole run after + its =, so the next character left is one that ends a value, never the value's first.""" + at = kept = 0 + while at < len(full): + if at and full[at - 1] == "=": + value = _QUERY_VALUE.match(full, at).group() + if value and not left.startswith(value, kept): + at += len(value) + continue + if kept >= len(left) or full[at] != left[kept]: + return False + at += 1 + kept += 1 + return kept == len(left) + + +# R's adversarial lone URLs (change review r1, b5.py): hosts that are names, ports that read as +# values, userinfo split by breaks, rule 2's query on the next line, and brackets. +_ADVERSARIAL_URLS = [ + f"https://{userinfo}{host}{port}{path}" + for userinfo in ("", "u:p@") + for host in ("token", "secret", "password", "a") + for port in (":8443", ":0", ":{PORT}", "") + for path in ("/a1", "") +] + [ + "https://u:pa\nss@h/x", + "https://u:pa\tss@h/x", + "https://u:p[a\nb@h/x", + "https://u:a]b\n c@h/x", + "https://u:12\nab@h", + "https://u:{P}\nab@h", + "https://u:ab\r\ncd@h/x?y=1", + "https://u:ab\ncd@token:8443/a1", + "redis://:pw\nx@cache:6379/0", + "https://u:p@h/token=a1b2", + "https://h/x\n?a=1", + "https://h/x\n&a=1/b", + "https://u:p@h/x\n&a=1/b", + "https://u:x\ny@h/a\tb", + "https://u:p@[fd00::5]:8443/x", +] + + +def test_leftovers_takes_only_a_userinfo_and_query_values_from_a_lone_url() -> None: + """cr1 L1, beside B5 and without hypothesis: rule 1 and rule 2 checked independently.""" + from fi.alk.providers.secret_shapes import leftovers + + taken = {url: leftovers(url) for url in _ADVERSARIAL_URLS} + + assert { + url: left + for url, left in taken.items() + if not _only_userinfo_and_query_values_taken(url, left) + } == {} + # Each rule takes something here, so the check is not passing on untouched input. + assert taken["https://u:pa\nss@h/x"] == "https://h/x" + assert taken["https://h/x\n?a=1"] == "https://h/x\n?a=" @st.composite diff --git a/tests/providers/test_secret_shapes.py b/tests/providers/test_secret_shapes.py new file mode 100644 index 000000000..33b5ce114 --- /dev/null +++ b/tests/providers/test_secret_shapes.py @@ -0,0 +1,1510 @@ +"""TH-9500: secrets found by their shape, in the kit's one cleaner (plan v1.1, tests S1-S5b). + +Every key-shaped value here is fake. The corpus below is the design reviewers' case lists, +copied as data (W2), so a planted bug in a rule fails a named case. +""" + +from __future__ import annotations + +import itertools +import logging +import re +import subprocess +import sys + +import pytest + +from fi.alk.providers import secret_shapes +from fi.alk.providers.agent import ( + _LEADING_SCHEME, + _REDACTED, + _SPACE, + _URL_BREAKS, + _redact, + _redact_token, +) + + +def _base(text: str) -> str: + """TH-9491's URL step alone, frozen here: what the cleaner did before TH-9500.""" + if "://" not in text: + return text + if _LEADING_SCHEME.match(text): + joined = text.translate(_URL_BREAKS) + if not _SPACE.search(joined): + return _redact_token(joined, []) + return "".join( + part if not part or part.isspace() else _redact_token(part, []) + for part in _SPACE.split(text) + ) + + +def _logged(caplog: pytest.LogCaptureFixture) -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if record.name == "fi.alk.providers.agent" + ] + + +def test_the_marker_and_the_held_bound_match_their_owners() -> None: + from fi.alk.harness.hosted_chat_entrypoint import _HOLD_HARD + + assert secret_shapes.REDACTED == _REDACTED + assert secret_shapes._LONGEST_HELD_KEY == _HOLD_HARD + + +# -- S1: the drift test, against the kit's guardrails list ------------------------------------ + +_SAMPLES = { + "openai_api_key": "sk-" + + "FAKEfake0123456789ab" + + "T3BlbkFJ" + + "FAKEfake0123456789ab", + "openai_api_key_generic": "sk-proj-" + + "FAKEfake0123456789abcdefFAKEfake0123456789ab", + "aws_access_key": "AKIA" + "FAKEFAKE12345678", + "google_api_key": "AIza" + "FAKEfake0123456789FAKEfake012345678", + "github_pat": "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8", + "github_oauth": "gho_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8", + "github_user_token": "ghu_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8", + "github_server_token": "ghs_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8", + "github_refresh_token": "ghr_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8", + "github_fine_grained_pat": "github_pat_" + + "FAKEfake0123456789abcd" + + "_" + + "F" * 59, + "gitlab_pat": "glpat-" + "FAKEfake0123456789ab", + "slack_token": "xoxa-1234567890-1234567890-" + "FAKEfakeFAKEfakeFAKEfake", + "slack_bot_token": "xoxb-12345678901-12345678901-" + "FAKEfakeFAKEfakeFAKEfake", + "stripe_secret_key": "sk_live_" + "FAKEfake0123456789FAKEfa", + "sendgrid_api_key": "SG." + + "FAKEfake0123456789abcd" + + "." + + "FAKEfake0123456789" * 2 + + "abcdefg", + "anthropic_api_key": "sk-ant-" + "FAKEfake0123456789" * 2 + "abcd", + "huggingface_token": "hf_" + "FAKEfake0123456789FAKEfake01234567", + "npm_token": "npm_" + "FAKEfake0123456789FAKEfake0123456789", + "pypi_token": "pypi-AgEIcHlwaS5vcmc" + "FAKEfake0123456789" * 3, + "private_key_header": "-----BEGIN RSA PRIVATE KEY-----\nMIIEFAKEQUJDREVG\n" + "-----END RSA PRIVATE KEY-----", + "pgp_private_key": "-----BEGIN PGP PRIVATE KEY BLOCK-----\nlQOYBFAKEQUJDREVG\n" + "-----END PGP PRIVATE KEY BLOCK-----", +} +# Every other guardrails entry at confidence 0.9 or more, with why it isn't a prefix pattern. +_LEFT_OUT = { + "aws_secret_key": "(?i) start; kept as its own pattern, from its own regex (r4 M3)", + "azure_storage_key": "(?i) start; kept as its own pattern, from its own regex (r4 M3)", + "firebase_api_key": "(?i) start; a name covered by the named-value rule, value AIza", + "mailchimp_api_key": "no literal start; left out on purpose, it matches honest hex IDs", + "discord_token": "no literal start; the bare Discord bot token is a named limit", + "telegram_bot_token": "no literal start; the bare Telegram token is our own pattern", + "database_url": "(?i) start; TH-9491's URL step removes the userinfo", + "connection_string": "(?i) start, and its .* makes the regex quadratic", +} + + +def _guardrails() -> dict[str, tuple[str, float]]: + from fi.evals.guardrails.scanners import secrets + + return { + name: (pattern, confidence) + for listed in vars(secrets).values() + if isinstance(listed, list) and listed and isinstance(listed[0], tuple) + for pattern, name, confidence in listed + } + + +def _literal_prefix(pattern: str) -> str: + out = [] + at = 0 + while at < len(pattern): + char = pattern[at] + if char == "\\" and at + 1 < len(pattern) and pattern[at + 1] in ".-/": + out.append(pattern[at + 1]) + at += 2 + continue + if char in "\\[](){}?*+|^$.": + break + out.append(char) + at += 1 + return "".join(out) + + +def test_the_prefix_patterns_and_the_left_out_entries_are_the_guardrails_list() -> None: + entries = {name: entry for name, entry in _guardrails().items() if entry[1] >= 0.9} + prefixed = { + name + for name, (pattern, _) in entries.items() + if len(_literal_prefix(pattern)) >= 3 + } + + assert prefixed == set(_SAMPLES) + assert len(prefixed) == 21 + assert set(entries) - prefixed == set(_LEFT_OUT) + + +@pytest.mark.parametrize("name", sorted(_SAMPLES)) +def test_each_prefix_patterns_sample_is_masked(name: str) -> None: + pattern, _ = _guardrails()[name] + sample = _SAMPLES[name] + assert re.search(pattern, sample) + + out = _redact(f"Use this:\n{sample}\nthen stop.") + + assert sample.splitlines()[-2 if "\n" in sample else 0][-12:] not in out + assert out.startswith("Use this:\n") and out.endswith("\nthen stop.") + + +def _name_forms(words: list[list[str]]) -> list[str]: + """Every join a guardrails name allows: each optional word in or out, each join "", "_" + or "-", in lower and upper case and as PascalCase.""" + forms = set() + for kept in itertools.product( + *[[word] if required else [word, ""] for word, required in words] + ): + parts = [part for part in kept if part] + for joins in itertools.product(["", "_", "-"], repeat=len(parts) - 1): + name = parts[0] + "".join( + join + part for join, part in zip(joins, parts[1:]) + ) + forms |= {name, name.upper()} + forms.add( + parts[0].title() + + "".join(join + part.title() for join, part in zip(joins, parts[1:])) + ) + return sorted(forms) + + +_AWS = "wJalrXUtnFEMIFAKEK7MDENGbPxRfiCYFAKEKEY1" +_AZURE = ("FAKEazureFAKEkey0123456789abcdefFAKE+/" * 3)[:86] + "==" +_FIREBASE = "AIza" + "FAKEfake0123456789FAKEfake012345678" +_NAMED_ENTRIES = { + "aws_secret_key": ( + [["aws", True], ["secret", True], ["access", False], ["key", True]], + _AWS, + ), + "azure_storage_key": ( + [["azure", True], ["storage", False], ["account", False], ["key", True]], + _AZURE, + ), + "firebase_api_key": ( + [["firebase", True], ["api", False], ["key", True]], + _FIREBASE, + ), +} + + +@pytest.mark.parametrize("entry", sorted(_NAMED_ENTRIES)) +def test_every_name_form_a_named_entry_accepts_is_masked(entry: str) -> None: + """The drift test for the named-value entries (r4 L5, M3): every name form the entry's + regex accepts, through this module.""" + pattern, _ = _guardrails()[entry] + words, value = _NAMED_ENTRIES[entry] + leaked = [] + for form in _name_forms([[word, required] for word, required in words]): + for join in ("=", " = ", ": "): + text = f"{form}{join}{value}" + assert re.search(pattern, text), text + if value in _redact(text): + leaked.append(text) + assert leaked == [] + + +@pytest.mark.parametrize("join", ["=", ": "]) +def test_the_bare_telegram_token_is_masked(join: str) -> None: + token = "123456789:AA" + "FAKEfake0123456789FAKEfake0123456" + assert re.search(_guardrails()["telegram_bot_token"][0], token) + assert _redact(f"BOT{join}{token} ok") == f"BOT{join}{_REDACTED} ok" + + +# Round 1 (r1cases.py), with its fake key shapes. +GH = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8" +SKP = "sk-proj-" + "FAKEfake0123456789abcdefFAKEfake0123456789ab" +AK = "AKIA" + "FAKEFAKE12345678" +XB = "xoxb-12345678901-12345678901-" + "FAKEfakeFAKEfakeFAKEfake" +SLACK = "https://hooks.slack.com/services/" + "T00FAKE00/B00FAKE00/" + "FAKEfakeFAKEfakeFAKEfake" +TG = "https://api.telegram.org/bot123456789:AAFAKEfakeFAKEfakeFAKEfakeFAKEfake_x/sendMessage" +JWT = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJmYWtlIn0.FAKEsigFAKEsig" +PEM = "-----BEGIN RSA PRIVATE KEY-----\nMIIEFAKEFAKEFAKE\nABCDEFFAKE==\n-----END RSA PRIVATE KEY-----" + +R1_CASES = { + "page: ws in password": "https://bot:hun ter2@h/v1", + "page: lb in password": "see https://bot:hun\n ter2@h/v1 is the hook", + "page: tab in password": "see https://bot:hun\tter2@h/v1 is the hook", + "page: query next line": "see https://u:pw@h/x\n ?key=sk-1 and more", + "page: ?# in password": "https://bot:8812#x@db/", + "page: / in password": "https://bot:pa/ss@db", + "page: :// in password": "https://bot:pa://ss@db", + "bare ghp prose": f"Use {GH} to push.", + "bare sk-proj prose": f"key is {SKP}.", + "bare AKIA quoted": f'aws="{AK}"', + "bare xoxb json": f'{{"t": "{XB}"}}', + "slack hook": f"Post to {SLACK} daily", + "telegram": f"Call {TG} now", + "jwt prose": f"Bearer {JWT}", + "pem": f"key:\n{PEM}\nend", + "pem header only": "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAAFAKE\n(truncated)", + "assign pw": "password=hunter22", + "assign spaced": "token: s3cr3tvalue99", + "tab query": "see https://h/x\t?key=sk-1 more", + "mid-query break": "see https://h/x?a=1&\n key=sk-1 more", + "break before ? no indent": "see https://h/x\n?key=sk-1 more", + "nbsp query": "see https://h/x ?key=sk-1 more", + "nbsp password": "https://bot:hun ter2@h/v1", + # honest text + "H port+email": "Run at http://localhost:3000 and email support@acme.com", + "H query email": "https://host:8080/path?email=a@b.com", + "H npm scoped": "Install from https://registry.npmjs.org:443/@babel/core today", + "H medium": "Read https://medium.com/@user for more", + "H port then mention": "Open http://localhost:8080 and ping @oncall in Slack", + "H md amp": "See https://docs.x.com/a\n©=2024 Acme", + "H password required": "password: required", + "H token expires": "the token: expires hourly", + "H secret never": "secret: never tell anyone", + "H env": 'api_key=os.environ["OPENAI_API_KEY"]', + "H env placeholder": "Send the token: ${SLACK_BOT_TOKEN} in the header", + "H commit": "Fixed in 1ac329e16b0f2c9a8e7d6c5b4a3f2e1d0c9b8a7f", + "H uuid": "order 123e4567-e89b-12d3-a456-426614174000", + "H b64img": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", + "H order": "Order ID ORD-2024-000123 and SKU AKIAMODEL", + "H token count": "max_tokens: 4096 and token=streaming", +} + +# Round 1's p2 and p5 additions (round 3's cases.py). +A = "FAKEfake0123456789" +R1_EXTRA = { + "r1p2 clone x:ghp": f"https://x:{GH}@github.com/org/repo.git", + "r1p5 anthropic": "sk-ant-api03-" + (A * 6)[:95], + "r1p5 google": "AIza" + (A * 2)[:35], + "r1p5 gitlab": "glpat-" + (A * 2)[:20], + "r1p5 hf": "hf_" + (A * 2)[:34], + "r1p5 BOT_TOKEN": "BOT_TOKEN=Zq8wYv3Lp9Kd", + "r1p5 yaml client_secret": "client_secret: Zq8wYv3Lp9Kd2", + "r1p5 PEM PKCS8": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASC\n-----END PRIVATE KEY-----", +} + +# Round 2 (p1.py). +R2_CASES = { + "owl p0 mid-query 2": "https://h/x?a=1&\n key=sk-1", + "owl p0 slack": "https://hooks.slack.com/services/" + "T00000000/B00000000/" + "XXXXXXXXXXXXXXXXXXXXXXXX", + "owl p0 token sk-proj-abc": "token: sk-proj-abc", + "H site residue": "Visit https://site.com:8443?ref=me@x.com", + "H port/ email": "Login at http://localhost:8080/login as admin@acme.com", + "r1 digits ws": "https://bot:12 34@h/v1", + "r1 digits slash": "https://bot:8812/x@db/", + "r1 two ws runs": "https://bot:hun ter 2@h", + "r1 user dot": "https://my.bot:hun ter2@h", + "r1 user dash": "https://ci-bot:hun ter2@h", + "r1 empty user redis": "redis://:pa/ss@cache:6379/0", + "r1 empty user ws": "redis://:hun ter2@cache:6379/0", + "H a.com:x y@z": "see http://a.com:x y@z now", + "H template port": "POST https://HOST:PORT/register?email=a@b.com", + "H template port2": "POST http://{HOST}:{PORT}/v1/users/@me", + "H ipv6 verdaccio": "npm i --registry http://[::1]:4873/@babel/core", + "r2 mid-value break": "https://api.x.com/v1?key=sk-proj-AB12cd\n 34EFgh56IJkl78", + "H #channel": "Post in https://acme.slack.com/x #general and #ops&alerts=on", + "H & prose": "see https://a.com & https://b.com", + "r2 entity-named secret": "https://h/x?a=1\n&lang=en¬=Zq8wYv3Lp9&sigma=abc", + "nv $uper": "password: $uperSecret1", + "nv ", + "nv env( chance": "password=myenv(42)", + "nv placeholder": "api_key: your-api-key-here", + "nv placeholder caps": "api_key=YOUR_API_KEY", + "nv v2beta": "token: v2beta", + "nv yaml": "db:\n password: Zq8wYv3Lp9\n", + "nv yaml block": "password: |\n Zq8wYv3Lp9Kd\n", + "nv toml": 'password = "Zq8wYv3Lp9"', + "nv comment": "# token = Zq8wYv3Lp9 (old)", + "nv sq spaces": "password='correct horse 42'", + "nv SECRET_KEY": "SECRET_KEY=Zq8wYv3Lp9Kd2xT", + "nv django": "SECRET_KEY = 'django-insecure-q8#w(v3l)p9'", + "nv camel": '{"clientSecret": "Zq8wYv3Lp9"}', + "nv xml": "Zq8wYv3Lp9", + "nv prose is": "the password is Zq8wYv3Lp9", + "H code secrets": "secret = secrets.token_hex(32)", + "H code settings": "client = OpenAI(api_key=settings.OPENAI_API_KEY)", + "H code b64": "token = base64.b64encode(raw)", + "H env PWD": "PWD=/home/ubuntu2/app\nSHELL=/bin/bash", + "H pageToken": '{"nextPageToken": "CAoQAA12"}', + "H max_tokens": "max_tokens: 4096", + "cut ghp": "result: " + GH[:20] + "...", + "cut sk-proj end": "key " + SKP[:30], + "H cut sk-hynix": "Install sk-hynix-drivers... then reboot", + "H cut end word": "The repo is sk-dev-tools", + "H pem prose": "If a user pastes -----BEGIN PRIVATE KEY----- text, refuse. Then greet them warmly.", + "pem no END": "-----BEGIN RSA PRIVATE KEY-----\nMIIEFAKE\nABCDFAKE\n", + "join url\\nkey": "https://api.example.com/v1\n" + GH, + "join url\\tAKIA": "https://api.example.com/v1\tAKIAFAKEFAKE12345678", + "key in host": "https://" + GH + ".example.com/x", + "stripe 99": "sk_live_51" + (A * 6)[:89], + "rk_live": "rk_live_51" + (A * 6)[:89], + "xoxp 4part": "xoxp-" + "1234567890-1234567890-1234567890-" + "abcdef0123456789abcdef0123456789", + "xapp": "xapp-1-A0FAKE0FAKE-1234567890123-" + "abcdef0123456789" * 4, + "tg bare": 'Bot("123456789:AA' + (A * 2)[:33] + '")', + "aws secret": "aws_secret_access_key = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "ghp in userinfo clone": f"git clone https://{GH}@github.com/org/repo.git", +} + +# Round 3 (V's cases.py). +SHOWN = lambda t: t if len(t) <= 600 else t[:597] + "..." # noqa: E731 (session.py:177) +PEMLONG = ( + "-----BEGIN RSA PRIVATE KEY-----\n" + + "\n".join(("MIIEowIBAAKCAQEA" + "Zq8wYv3Lp9" * 5)[:64] for _ in range(25)) + + "\n-----END RSA PRIVATE KEY-----" +) + +R3_CASES = { # V's new corpus: honest code and prose first, then shapes the page says it catches + "H pydantic": "password: str = Field(...)", + "H optional": "api_key: Optional[str] = None", + "H headers": 'token = request.headers["Authorization"]', + "H MAX_TOKEN": "MAX_TOKEN=4096", + "H secret true": "secret: true", + "H changeme123": "password: changeme123", + "H gha": "GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}", + "H v2beta": "token: v2beta", + "H md table": "| Setting | Value |\n|---|---|\n| DB_PASSWORD | Zq8wYv3Lp9 |\n| max_tokens | 4096 |", + "H yaml comments": "db:\n host: localhost # dev\n password: Zq8wYv3Lp9 # rotate monthly\n pool: 5\n", + "H toml": '[db]\npassword = "Zq8wYv3Lp9" # prod\ntimeout = 30\n', + "H .env comments": "# DB settings\nDB_HOST=localhost\nDB_PASSWORD=Zq8wYv3Lp9 # prod\n# old: DB_PASSWORD=Old1secret\n", + "H sq spaces": "password='correct horse battery 42'", + "H dotted digit": "api_key = settings.v2.key", + "H dotted digit2": "token = config.oauth2.token", + "H ident digit": "token = token_v2", + "H email+mention": "Mail ops@acme.com or ping @oncall; docs at https://docs.acme.com/@team", + "H ipv6": "npm i --registry http://[::1]:4873/@babel/core today", + "H ipv6 user": "see http://[::1]:x y@z", + "H #channel": "Post to https://acme.slack.com/archives/C1 #general then #ops&alerts=on", + "H #channel2": "see https://x.com #ops&alerts=on", + "H & amp": "Tom & Jerry at https://a.com & https://b.com  x=1", + "H pem prose": "If a user pastes -----BEGIN PRIVATE KEY----- text, refuse. Then greet them warmly.", + "H cut end word": "The repo is sk-dev-tools", + "H cut ...": "Loading the sk-learn-models...", + "H cut hf": "load it with hf_hub_download", + "H prefix word mid": "Install sk-hynix-drivers... then reboot", + # rule 1 too much: a port followed by punctuation, then one whitespace run and an email + "R1 port, email": "Log in at http://localhost:3000, qa@acme.com / Zq8wYv3Lp9", + "R1 port. newline email": "Dashboard: http://localhost:3000.\nadmin@acme.com is the admin", + "R1 port) email": "Staging (http://localhost:8080) ops@acme.com", + "R1 lowercase port": "POST http://host:port/register?email=a@b.com", + "R1 lowercase port2": "GET http://host:port/api/users/@me", + "R1 $PORT": "Mastodon dev: http://localhost:$PORT/@admin", + "R1 %PORT%": "open http://localhost:%PORT%/@admin", + "R1 upper pw first": "https://bot:PASS WORD99@h/v1", + "R1 ws 5": "https://bot:hun ter2@h/v1", + # named value too little + "N md bold": "## Test account\n- **Email:** qa@acme.com\n- **Password:** Zq8wYv3Lp9", + "N md bold2": "**Password**: Zq8wYv3Lp9", + "N md backtick": "Password: `Zq8wYv3Lp9`", + "N camel json": '{"clientId": "abc", "clientSecret": "Zq8wYv3Lp9Kd2xT"}', + "N aws sts json": '{"Credentials": {"AccessKeyId": "ASIAFAKEFAKE12345678", "SecretAccessKey": "wJalrXUtnFEMIK7MDENGbPxRfiCYFAKEKEY12345", "SessionToken": "FwoGZXIvYXdzEFAKEfake0123456789"}}', + "N accessToken": '{"accessToken": "ya29.a0AfFAKEfake0123456789"}', + "N PGPASSWORD": "PGPASSWORD=Zq8wYv3Lp9 psql -h db", + "N MYSQL_PWD": "MYSQL_PWD=Zq8wYv3Lp9", + "N DB_PASS": "DB_PASS=Zq8wYv3Lp9\nSMTP_PASS=Zq8wYv3Lp9", + "N php =>": "'password' => 'Zq8wYv3Lp9',", + "N go :=": 'password := "Zq8wYv3Lp9"', + "N cli flag": "docker login -u bot --password Zq8wYv3Lp9 registry.acme.com", + "N bearer": 'curl -H "Authorization: Bearer Zq8wYv3Lp9Kd2xTq7" https://api.acme.com', + "N yaml phrase": "password: correct horse battery staple", + "N url\\npassword": "https://api.example.com/v1\npassword=Zq8wYv3Lp9", + "N url\\nDB_PASSWORD": "https://api.example.com/v1\nDB_PASSWORD=Zq8wYv3Lp9", + # key shapes + "K discordapp": "Post to https://discordapp.com/api/webhooks/123456789012345678/" + + (A * 4)[:68], + "K discord": "Post to https://discord.com/api/webhooks/123456789012345678/" + + (A * 4)[:68], + "K akia host": "https://AKIAFAKEFAKE12345678.example.com/x", + "K pem crlf no END": "-----BEGIN PRIVATE KEY-----\r\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASC\r\nZq8wYv3Lp9Zq8wYv3Lp9\r\n", + "K pem 600 cut": SHOWN("cat key.pem\n" + PEMLONG), + "K pem 600 cut short": SHOWN( + "x" * 560 + + "\n-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIBZq8wYv3Lp9Zq8wYv3Lp9Zq8wYv3Lp9Zq8wYv3Lp9oAoGCCqGSM49" + ), + "K pem json cut": SHOWN( + '{"type": "service_account", "private_key": "-----BEGIN PRIVATE KEY-----\\n' + + "\\n".join(["MIIEvQIBADANBgkqhkiG9w0BAQEFAASC" * 2] * 20) + + '\\n-----END PRIVATE KEY-----\\n"}' + ), + "K pem single line": "PRIVATE_KEY=-----BEGIN PRIVATE KEY-----MIIEvQIBADANBgkqhkiG9w0BAQEFAASCZq8w", + "K ghp cut": SHOWN("x " * 290 + GH), + "K tg cut": SHOWN( + "y " * 280 + + "see https://api.telegram.org/bot123456789:AAFAKEfakeFAKEfakeFAKEfakeFAKEfake_x/sendMessage" + ), +} + +# The page's own outcomes (round 3): None means TH-9491's result stands. +PAGE_OUTCOMES = { # the page's own words, as (input -> output); None = TH-9491's result stands + "https://bot:hun ter2@h/v1": "https://h/v1", + "https://bot:pa/ss@db": "https://db", + "Run at http://localhost:3000 and email support@acme.com": None, + "Open http://localhost:8080 and ping @oncall": None, + "POST https://HOST:PORT/register?email=a@b.com": None, + "http://{HOST}:{PORT}/v1/users/@me": None, + "https://bot:8812#x@db/": None, + "https://bot:8812/x@db/": None, + "https://bot:12 34@h": None, + "https://bot:hun ter 2@h": None, + "See https://docs.x.com/a\n©=2024 Acme": None, + "https://api.example.com/v1\n" + GH: "https://api.example.com/v1[redacted]", + f"git clone https://{GH}@github.com/org/repo.git": "git clone https://github.com/org/repo.git", + "https://" + GH + ".example.com/x": "[redacted]", + "If a user pastes -----BEGIN PRIVATE KEY----- text, refuse. Then greet them warmly.": "If a user pastes [redacted] text, refuse. Then greet them warmly.", + "password: required": None, + "token=streaming": None, + 'api_key=os.environ["OPENAI_API_KEY"]': None, + "secret = secrets.token_hex(32)": None, + "token = base64.b64encode(raw)": None, + "client = OpenAI(api_key=settings.OPENAI_API_KEY)": None, + "PWD=/home/ubuntu2/app\nSHELL=/bin/bash": None, + '{"nextPageToken": "CAoQAA12"}': None, + '{"password": "Zq8wYv3Lp9"}': '{"password": "[redacted]"}', + "DB_PASSWORD=Zq8wYv3Lp9": "DB_PASSWORD=[redacted]", + "SECRET_KEY=Zq8wYv3Lp9Kd2xT": "SECRET_KEY=[redacted]", + "SECRET_KEY = 'django-insecure-q8#w(v3l)p9'": "SECRET_KEY = '[redacted]'", + "password: $uperSecret1": None, + "Zq8wYv3Lp9": None, + "the password is Zq8wYv3Lp9": None, + "Install sk-hynix-drivers... then reboot": None, + "token: v2beta": "token: [redacted]", + "api_key: your-api-key-here": "api_key: [redacted]", +} + +# Round 4 (J's jcases.py). +AWS = "wJalrXUtnFEMIFAKEK7MDENGbPxRfiCYFAKEKEY1" +B64 = "MIIEowIBAAKCAQEAzq8wYv3Lp9Kd2xTq7FAKEfake0123456789abcdefFAKEfake" * 40 +PEMB = "\n".join(B64[i : i + 64] for i in range(0, 1600, 64)) + +R4_HONEST = { # want: same as TH-9491 alone + "pydantic Field": "password: str = Field(...)", + "Optional": "api_key: Optional[str] = None", + "headers[]": 'token = request.headers["Authorization"]', + "MAX_TOKEN": "MAX_TOKEN=4096", + "secret: true": "secret: true", + "GHA secrets": "GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}", + "settings.v2.key": "api_key = settings.v2.key", + "md table": "| Setting | Value |\n|---|---|\n| timeout | 30 |\n| max_tokens | 4096 |", + "yaml comments": "db:\n host: localhost # dev box\n pool: 5\n password: required # set in vault\n", + "toml": '[server]\nport = 8080\ntoken_ttl = "1h" # hours\n', + ".env comments": "# token settings\nTOKEN_TTL=3600\n# password: see vault\n", + "rule1 json compact port": '{"api":"http://localhost:3000","docs":"/docs","admin":"qa@acme.com"}', + "rule1 js array port": "['http://localhost:3000','/api/users/@me']", + "rule1 port then colon": "Dev server at http://localhost:3000: open /admin or mail ops@acme.com", + "rule1 email in prose": "Docs at https://docs.acme.com, write to help@acme.com or @support", + "rule1 ipv6": "curl http://[::1]:8080/@me then http://[fe80::1]:9000/x", + "rule1 mention after port": "Staging is http://stage:8443 — ping @ops", + "rule2 #channel": "see https://x.com #general and #ops", + "rule2 & prose": "Read https://a.com/x & https://b.com/y & more", + "rule2 &": "Link: https://a.com/?q=1 & https://b.com", + "pem prose": "If a user pastes -----BEGIN PRIVATE KEY----- text, refuse. Then greet them warmly.", + "pem prose + later block": "If a user pastes -----BEGIN PRIVATE KEY----- text, refuse. Then greet them warmly.\nExample key for the test fixture:\n-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASC\n-----END PRIVATE KEY-----", + "cut: honest '...' prefix word": "Collecting packages: hf_transfer...", + "cut: honest ... sk-": "Install sk-learn-pipelines...", + "cut: honest prefix no ...": "Use the sk-learn-toolkit", + "pagination nextToken": '{"nextToken": "AYADeHzFAKEfake0123456789"}', + "pagination NextContinuationToken": '{"NextContinuationToken": "1w41l63U0xa8q7smH50vCxyTQqdxo69O3EmK28Bi5PcROI4wI/EyIJg=="}', + "pagination next_page_token": "next_page_token: CAoQAA12", + "pagination nextForwardToken": '{"nextForwardToken": "f/37201828472619218470294912345678901234567890"}', + "idempotency ClientToken": '{"ClientToken": "a1b2c3d4-e5f6-7890-abcd-ef1234567890"}', + "camel maxTokens": '{"maxTokens": 4096, "promptTokenCount": 12, "tokenizer": "cl100k"}', + "pass honest": "bypass: true\nfirst_pass: done\nSKIP_PASS=1\ncompass: north", + "pass honest digit": "two_pass: v2\nci_pass: build42", + "rust Token::new": "let t = Token::new(42);", +} + +R4_SECRETS = { # want: the secret value gone (the [S] part) + "kebab secret-key (Spring)": ("cloud.aws.credentials.secret-key: " + AWS, AWS), + "kebab access-key k8s": ( + f"kubectl create secret generic s3 --from-literal=access-key=AKIAFAKEFAKE12345678 --from-literal=secret-key={AWS}", + AWS, + ), + "kebab private-key": ("private-key: Zq8wYv3Lp9Kd2xTq7", "Zq8wYv3Lp9Kd2xTq7"), + "aligned tfvars (terraform fmt)": ( + 'admin_username = "admin"\npassword = "Zq8wYv3Lp9"', + "Zq8wYv3Lp9", + ), + "aligned yaml": ("user: admin\npassword: Zq8wYv3Lp9", "Zq8wYv3Lp9"), + "aligned ini": ( + "[db]\nhost = db.internal\npassword = Zq8wYv3Lp9\napi_key = Zq8wYv3Lp9Kd2xTq7", + "Zq8wYv3Lp9Kd2xTq7", + ), + "dotted password": ("DB_PASSWORD=Acme.Admin1", "Acme.Admin1"), + "dotted random": ("password=x7.Kq9vL2mPz", "x7.Kq9vL2mPz"), + "ya29 unquoted": ( + "GOOGLE_ACCESS_TOKEN=ya29.a0AfB_byCFAKEfake0123456789abcdefFAKE", + "ya29.a0AfB_byCFAKEfake0123456789abcdefFAKE", + ), + "ya29 quoted": ( + '{"access_token": "ya29.a0AfB_byCFAKEfake0123456789abcdefFAKE"}', + "ya29.a0AfB_byCFAKEfake0123456789abcdefFAKE", + ), + "yaml block scalar": ("password: |\n Zq8wYv3Lp9Kd\n", "Zq8wYv3Lp9Kd"), + "unquoted passphrase": ("password: correct horse battery 42", "battery 42"), + "rule1 trailing space before break": ( + "hook: https://bot:hun \n ter2@h/v1", + "ter2", + ), + "rule1 blank line": ("hook: https://bot:hun\n\n ter2@h/v1", "ter2"), + "pem END-less escaped JSON, no name": ( + SHOWN( + '{"key": "-----BEGIN RSA PRIVATE KEY-----\\n' + + PEMB.replace("\n", "\\n") + + '\\n-----END RSA PRIVATE KEY-----"}' + ), + B64[:40], + ), + "pem END-less flattened spaces": ( + SHOWN( + "TLS_KEY=-----BEGIN RSA PRIVATE KEY----- " + + PEMB.replace("\n", " ") + + " -----END RSA PRIVATE KEY-----" + ), + B64[:40], + ), + "pem END-less encrypted, cut": ( + SHOWN( + "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-128-CBC,3F2A9C1B7E4D6F80A1B2C3D4E5F60718\n\n" + + PEMB + + "\n-----END RSA PRIVATE KEY-----" + ), + B64[:40], + ), + "pem END-less Read tool + arrow": ( + SHOWN( + "\n".join( + f"{i + 1:6}→{line}" + for i, line in enumerate( + ("-----BEGIN RSA PRIVATE KEY-----\n" + PEMB).split("\n") + ) + ) + ), + B64[:40], + ), + "camel passwd/pwd": ( + '{"dbPwd": "Zq8wYv3Lp9", "rootPasswd": "Zq8wYv3Lp9"}', + "Zq8wYv3Lp9", + ), + "pass camel": ('{"smtpPass": "Zq8wYv3Lp9"}', "Zq8wYv3Lp9"), +} + +AZ = ("FAKEazureFAKEkey0123456789abcdefFAKE+/" * 3)[:86] + "==" +FB = "AIza" + "FAKEfake0123456789FAKEfake012345678" + +R4_DRIFT = [ + ("aws_secret_key", f"aws_secret_key = {AWS}"), + ("aws_secret_key", f"AWS_SECRET_ACCESS_KEY={AWS}"), + ("aws_secret_key", f"aws-secret-key: {AWS}"), + ("aws_secret_key", f"awssecretkey={AWS}"), + ("aws_secret_key", f"AWSSecretKey={AWS}"), + ("azure_storage_key", f"AZURE_STORAGE_KEY={AZ}"), + ("azure_storage_key", f"azure_key = {AZ}"), + ("azure_storage_key", f"azure-storage-key: {AZ}"), + ("azure_storage_key", f"AzureStorageAccountKey={AZ}"), + ("firebase_api_key", f"FIREBASE_API_KEY={FB}"), + ("firebase_api_key", f"firebasekey: {FB}"), + ("mailchimp_api_key", "MC key 0123456789abcdef0123456789abcdef" + "-us12"), + ( + "discord_token", + "MTAxNTI3OTM4NjQ5Nzc0MDAzMg" + ".GFAKEf." + "FAKEfake0123456789FAKEfake012", + ), + ("telegram_bot_token", "123456789:AAFAKEfakeFAKEfakeFAKEfakeFAKEfake_x"), + ("database_url", "postgres://app:Zq8wYv3Lp9@db.internal:5432/app"), + ("connection_string", "Server=db;Database=app;User Id=sa;Pwd=Zq8wYv3Lp9;"), + ("connection_string", "Host=db;Username=app;Password=Zq8wYv3Lp9;"), +] + + +# -- S2: the design page's cases, rule by rule -------------------------------------------------- + +_B64 = "MIIEowIBAAKCAQEAzq8wYv3Lp9Kd2xTq7FAKEfake0123456789abcdefFAKEfake" * 40 +_PEM_JSON = "\\n".join(_B64[i : i + 64] for i in range(0, 1600, 64)) +_SHOWN = lambda text: text if len(text) <= 600 else text[:597] + "..." # noqa: E731 +_GH = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8" +_PW = "Zq8wYv3Lp9" + +_MASKED = { + # Rule 1: a password broken by whitespace, /, ? or #, with up to 64 characters a side. + "rule1 space": ("https://bot:hun ter2@h/v1", "https://h/v1"), + "rule1 break + indent": ( + "see https://bot:hun\n ter2@h/v1 is the hook", + "see https://h/v1 is the hook", + ), + "rule1 slash": ("https://bot:pa/ss@db", "https://db"), + "rule1 empty user": ("redis://:hun ter2@cache:6379/0", "redis://cache:6379/0"), + "rule1 64 a side": ("https://bot:" + "a" * 64 + " b@h/v1", "https://h/v1"), + # Rule 2: a query after a break. + "rule2 next line": ( + "see https://u:pw@h/x\n ?key=sk-1 and more", + "see https://h/x\n ?key= and more", + ), + "rule2 query ends in &": ( + "see https://h/x?a=1&\n key=sk-1 more", + "see https://h/x\n key= more", + ), + # A named value: up to 16 spaces or tabs of alignment, or up to 4 whitespace with a break. + "named json": ('{"password": "Zq8wYv3Lp9"}', '{"password": "[redacted]"}'), + "named env": ("DB_PASSWORD=Zq8wYv3Lp9", "DB_PASSWORD=[redacted]"), + "named django": ( + "SECRET_KEY = 'django-insecure-q8#w(v3l)p9'", + "SECRET_KEY = '[redacted]'", + ), + "named 16 spaces": ( + "password" + " " * 16 + "= " + _PW, + "password" + " " * 16 + "= [redacted]", + ), + "named break after :": ("password:\n " + _PW, "password:\n [redacted]"), + "named space break after :": ("password: \n " + _PW, "password: \n [redacted]"), + "named break after =": ("password=\n " + _PW, "password=\n [redacted]"), + "named break before :": ("password\n : " + _PW, "password\n : [redacted]"), + "named break before =": ("password\n = " + _PW, "password\n = [redacted]"), + "named md bold label": ("- **Password:** " + _PW, "- **Password:** [redacted]"), + "named md bold name": ("**Password**: " + _PW, "**Password**: [redacted]"), + "named kebab": ( + "cloud.aws.credentials.secret-key: " + _PW, + "cloud.aws.credentials.secret-key: [redacted]", + ), + "named camel": ( + '{"clientSecret": "Zq8wYv3Lp9Kd2xT"}', + '{"clientSecret": "[redacted]"}', + ), + "named DB_PASS": ("DB_PASS=" + _PW, "DB_PASS=[redacted]"), + # An upper-case name joined by "-" (r4 L2): "-" is a join, not only a boundary. + "named DB-PASS": ("DB-PASS=" + _PW, "DB-PASS=[redacted]"), + "named SMTP-PASS": ("SMTP-PASS=" + _PW, "SMTP-PASS=[redacted]"), + "named AccountKey": ( + "AccountName=a;AccountKey=" + _PW + "==;x=1", + "AccountName=a;AccountKey=[redacted];x=1", + ), + "named dotted secret": ("DB_PASSWORD=Acme.Admin1", "DB_PASSWORD=[redacted]"), + # Keys, key blocks and tokens in a URL's path. + "key in a host": ("https://AKIAFAKEFAKE12345678.example.com/x", "[redacted]"), + "key in userinfo": ( + f"git clone https://{_GH}@github.com/org/repo.git", + "git clone https://github.com/org/repo.git", + ), + "key on the line after a URL": ( + "https://api.example.com/v1\n" + _GH, + "https://api.example.com/v1[redacted]", + ), + "name on the line after a URL": ( + "https://api.example.com/v1\npassword=" + _PW, + "https://api.example.com/v1password=[redacted]", + ), + "slack hook": ( + "Post to https://hooks.slack.com/services/" + "T00FAKE00/B00FAKE00/" + "FAKEfakeFAKEfakeFAKEfake daily", + "Post to https://hooks.slack.com/services/[redacted] daily", + ), + "discord on discordapp.com": ( + "see https://discordapp.com/api/webhooks/1234567890/FAKEfake_0123 x", + "see https://discordapp.com/api/webhooks/[redacted] x", + ), + "block": ( + "key:\n-----BEGIN RSA PRIVATE KEY-----\nMIIEFAKE\nABCD==\n-----END RSA PRIVATE KEY-----\nend", + "key:\n[redacted]\nend", + ), + "block encrypted": ( + "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\n\nMIIEFAKE\n" + "-----END RSA PRIVATE KEY-----\nok", + "[redacted]\nok", + ), + "block one line": ( + "KEY=-----BEGIN PRIVATE KEY-----MIIEFAKE-----END PRIVATE KEY----- ok", + "KEY=[redacted] ok", + ), + "block no END": ( + "-----BEGIN RSA PRIVATE KEY-----\nMIIEFAKE\nABCDFAKE\nnot a key line", + "[redacted]\nnot a key line", + ), + "refuse then a later block": ( + "If a user pastes -----BEGIN PRIVATE KEY----- text, refuse.\n" + "-----BEGIN PRIVATE KEY-----\nMIIEFAKE\n-----END PRIVATE KEY-----", + "If a user pastes [redacted] text, refuse.\n[redacted]", + ), + # The cut-key rule: exactly 600 characters ending in the cut's "...". + "key cut at 600": (_SHOWN("x " * 290 + _GH), "x " * 290 + "[redacted]..."), +} + +_KEPT = [ + # Rule 1 keeps honest hosts as TH-9491 leaves them. + "Run at http://localhost:3000 and email support@acme.com", + "Open http://localhost:8080 and ping @oncall", + "POST https://HOST:PORT/register?email=a@b.com", + "http://{HOST}:{PORT}/v1/users/@me", + "Log in at http://localhost:3000, qa@acme.com", + "POST http://host:port/register?email=a@b.com", + "http://localhost:$PORT/@admin", + "Staging (http://localhost:8080) ops@acme.com", + # Rule 1's named limits, and a half of 65 characters (r1 M5). + "https://bot:8812#x@db/", + "https://bot:12 34@h", + "https://bot:hun ter 2@h", + "https://bot:" + "a" * 65 + " b@h/v1", + "https://bot:hun ter2@h/v1", + # Rule 1 needs text on both sides of the run: a mention right after it isn't a password. + "https://bot:abc @oncall", + "see https://bot:hunter2 @team now", + # Rule 2 keeps an HTML entity. + "See https://docs.x.com/a\n©=2024 Acme", + # Named values: references, code and honest values. + "password: required", + "token=streaming", + 'api_key=os.environ["OPENAI_API_KEY"]', + "secret = secrets.token_hex(32)", + "token = base64.b64encode(raw)", + "client = OpenAI(api_key=settings.OPENAI_API_KEY)", + "api_key = settings.v2.key", + "PWD=/home/ubuntu2/app\nSHELL=/bin/bash", + '{"nextPageToken": "CAoQAA12"}', + "two_pass: v2", + "let t = Token::new(42);", + "password: $uperSecret1", + "Zq8wYv3Lp9", + "the password is Zq8wYv3Lp9", + "password" + " " * 17 + "= " + _PW, + # A quoted value for each reference form (r1 M5). + 'token: "{{ secrets.TOKEN_1 }}"', + 'token: "$TOKEN_1"', + 'token: "${TOKEN_1}"', + 'token: ""', + 'token: "os.environ.get(TOKEN_1)"', + 'token: "process.env.TOKEN_1"', + 'token: "env(TOKEN_1)"', + # A break with 5 whitespace characters on either side of the = or : (r1 M2). + "password:\n " + _PW, + "password=\n " + _PW, + "password\n : " + _PW, + "password\n = " + _PW, + # The cut-key rule needs both the length and the "..." (r4 L1). + "Collecting packages: hf_transfer...", + "Install sk-hynix-drivers... then reboot", + "The repo is sk-dev-tools", + # 601 characters, so not the cut. + "x " * 289 + _GH[:20] + "...", +] + + +@pytest.mark.parametrize("name", sorted(_MASKED)) +def test_the_pages_masked_cases(name: str) -> None: + text, expected = _MASKED[name] + assert _redact(text) == expected + + +# Written out, not read from the module: a name dropped there must fail here. +_PAGINATION_NAMES = ( + "ClientToken", + "NextContinuationToken", + "NextToken", + "continuationToken", + "nextPageToken", + "nextToken", + "pageToken", +) + + +@pytest.mark.parametrize("name", _PAGINATION_NAMES) +def test_every_pagination_name_keeps_its_value(name: str) -> None: + """Each "not a name" is kept, not only the ones the page quotes.""" + text = '{"%s": "CAoQAA12"}' % name + assert _redact(text) == text + + +@pytest.mark.parametrize("text", _KEPT, ids=range(len(_KEPT))) +def test_the_pages_kept_cases_come_out_as_th9491_leaves_them(text: str) -> None: + assert _redact(text) == _base(text) + + +# cr1 M1 (change review r1, R's hostname.py): a host whose name ends in a name, with a port, +# as Compose and Kubernetes service names do. Its ":" opens no value, so the URL stays as +# TH-9491 leaves it. +_HOST_NAMES = [ + "Call the issuer at http://auth-token:8080/v1/issue before each run.", + "Vault runs at http://vault-secret:8200/v1/kv in compose.", + "The token service: http://token:9000/oauth/token", + "GET http://api-key:3000/keys then POST", + "Use http://secret:8080 for local dev.", + "Use http://my-password:5000/reset to reset.", + "postgres://app@db-password:5432/app", + "see http://Auth-Token:8080/v1 now", + "base_url: http://token-svc:8080/v1", + "http://auth-token.internal:8080/v1", +] + + +@pytest.mark.parametrize("text", _HOST_NAMES, ids=range(len(_HOST_NAMES))) +def test_a_host_named_like_a_secret_keeps_its_port_and_path(text: str) -> None: + assert _redact(text) == _base(text) + + +def test_a_named_value_after_a_url_in_one_run_is_still_masked() -> None: + """cr1 M1: only a port after the name keeps it; this "=" opens a value.""" + out = _redact("url=https://api.x.com,token=Zq8wYv3Lp9Kd next words") + + assert "zq8wyv3lp9kd" not in out.lower() + assert out.endswith(" next words") + + +# -- Rule 1b: an @ before the password's / (design addendum v2.5.2) ----------------------------- +# Each cleaned row as (input, output). The chat stream runs these same rows at every split +# (test_builder_chat.py), and outbound runs the first four (test_outbound.py). +R1B_CLEANED = { + # Dileep's #278 threads P1 and P3: a password holding / or #, rule 1's own. + "Dileep P1": ( + "postgres://admin:Xk9/Qp+Zr8@db.example.com:5432/app", + "postgres://db.example.com:5432/app", + ), + "Dileep P3": ("https://user:pa#ss@host.example/x", "https://host.example/x"), + # Ash's two: an @ in the password, and an @ in the user. + "Ash P@ss": ( + "postgres://admin:P@ss/Xk9@db.example.com:5432/app", + "postgres://db.example.com:5432/app", + ), + "Ash smtp": ( + "smtp://me@gmail.com:app/pass@smtp.gmail.com:587", + "smtp://smtp.gmail.com:587", + ), + "redis": ("redis://:p@w/x9@cache.internal:6379/0", "redis://cache.internal:6379/0"), + ":99": ( + "postgres://admin:P@ss:99/Xk9@db.example.com/app", + "postgres://db.example.com/app", + ), + # M2's three hosts. + "localhost": ( + "postgres://admin:P@ss/Xk9@localhost/app", + "postgres://localhost/app", + ), + "a single label": ("postgres://admin:P@ss/Xk9@db/app", "postgres://db/app"), + "IPv6 with a port": ( + "postgres://admin:P@ss/Xk9@[::1]:5432/app", + "postgres://[::1]:5432/app", + ), + # M3 and M-A: a one-URL value runs step 1 on its joined token. + "M3 broken at Xk\\n9": ( + "postgres://admin:P@ss/Xk\n9@db.example.com:5432/app", + "postgres://db.example.com:5432/app", + ), + "M-A broken at Pa\\nss": ( + "postgres://admin:Pa\nss@w/x9@db.example.com/app", + "postgres://db.example.com/app", + ), + "M-A redis": ( + "redis://:pa\nss@w/x9@cache.internal:6379/0", + "redis://cache.internal:6379/0", + ), + # M-B's hosts. + "M-B postgres_db:5432": ( + "postgres://admin:P@ss/Xk9@postgres_db:5432/app", + "postgres://postgres_db:5432/app", + ), + "M-B my_db/app": ("postgres://admin:P@ss/Xk9@my_db/app", "postgres://my_db/app"), + "M-B a backtick": ("`redis://:p@w/x9@redis`", "`redis://redis`"), + "M-B db:${DB_PORT}": ( + "postgres://admin:P@ss/Xk9@db:${DB_PORT}/app", + "postgres://db/app", + ), + "M-B db:$DB_PORT": ( + "postgres://admin:P@ss/Xk9@db:$DB_PORT/app", + "postgres://db/app", + ), + "M-B db?sslmode": ("postgres://admin:P@ss/Xk9@db?sslmode=disable", "postgres://db"), + "M-B redis#0": ("redis://:p@w/x9@redis#0", "redis://redis"), + "M-B )": ("(amqp://u:p@w/x9@rabbitmq)", "(amqp://rabbitmq)"), + "M-B ,": ("amqp://u:p@w/x9@rabbitmq, then", "amqp://rabbitmq, then"), + "M-B ;": ("amqp://u:p@w/x9@rabbitmq; then", "amqp://rabbitmq; then"), + "M-B >": ("", ""), + # TH-9491 refuses the stray ], as it would with no password. + "M-B ]": ("[amqp://u:p@w/x9@rabbitmq]", "[[redacted]"), + "M-B zoned IPv6": ( + "postgres://admin:P@ss/Xk9@[fe80::1%25en0]:5432/app", + "postgres://[fe80::1%25en0]:5432/app", + ), + # M4: one sorted, merged list of rule 1's and 1b's spans. + "M4 1b then rule 1": ( + "see postgres://admin:P@ss/Xk9@db.example.com/app and https://bot:hun ter2@h/v1 now", + "see postgres://db.example.com/app and https://h/v1 now", + ), + "M4 rule 1 then 1b": ( + "see https://bot:hun ter2@h/v1 and postgres://admin:P@ss/Xk9@db.example.com/app now", + "see https://h/v1 and postgres://db.example.com/app now", + ), + "M4 overlapping": ("a://u:x://v:p@h/x@host.example", "a://host.example"), +} +# M1 and M-C: a user with no password keeps its host, and a path's @ before no host is left +# alone, as TH-9491 leaves them. +R1B_KEPT = [ + "http://admin@localhost:8080/bob@example.com", + "ssh://git@gitlab.example.com:2222/ops@infra.example/repo.git", + "https://ci@registry.example.com:8443/lodash@4.17.21", + "ssh://git@host.example:22/org@team/repo", + "Read https://medium.com/@user for more", + "Install from https://registry.npmjs.org:443/@babel/core today", + "http://admin@[::1]/bob@example.com", + "ssh://git@[fd00::5]/org@team/repo", + "http://admin@[::1]:8080/bob@example.com", + # A user with no : at all. + "https://ci@registry.example.com/lodash@4.17.21", + "ssh://git@github.com/org@team/repo", + # A path's @ with no host after it: rule 1b doesn't hold, and TH-9491 removes the userinfo. + "Mail https://u:p@h.example/to@~team now", +] +# The addendum's named limits, pinned as they come out. +R1B_LIMITS = { + # L1: the host moves; the password is still removed. + "https://u:p@api.example.com/a@b.com": "https://b.com", + "https://u:p@api.example.com/lodash@4.17.21": "https://4.17.21", + "https://u:p@api.example.com/react@18?dev": "https://18", + "git+https://u:${TOKEN}@git.example.com/repo.git@main": "git+https://main", + # L2: the scp form inside a scheme. + "git+ssh://git@github.com:org/repo@v1.2.0": "git+ssh://v1.2.0", + "git+ssh://git@github.com:org/repo@main": "git+ssh://main", + # M-C: a port rule 1 doesn't read. + "http://admin@localhost:{{port}}/bob@example.com": "http://example.com", + "http://admin@localhost:$(PORT)/bob@example.com": "http://example.com", + "http://admin@localhost:/bob@example.com": "http://example.com", + # Still leaks, as on the base: a password that reads as a port, an empty segment, two /, + # a segment over its bound, %40, and a line break inside the span in prose. + "smtp://me@gmail.com:1234/pass@smtp.gmail.com": "smtp://gmail.com:1234/pass@smtp.gmail.com", + "postgres://admin:P@ss/@db.example.com/app": "postgres://ss/@db.example.com/app", + "postgres://admin:P@ss/a/b@db.example.com/app": "postgres://ss/a/b@db.example.com/app", + "postgres://admin:P@ss/" + "a" * 65 + "@db.example.com/app": "postgres://ss/" + + "a" * 65 + + "@db.example.com/app", + "postgres://admin:P%40ss/Xk9@db.example.com/app": "postgres://admin/Xk9@db.example.com/app", + "see postgres://admin:P@ss/Xk\n9@db.example.com/app now": "see postgres://ss/Xk\n9@db.example.com/app now", +} + + +@pytest.mark.parametrize("name", list(R1B_CLEANED)) +def test_rule_1b_cleans_the_addendums_rows(name: str) -> None: + text, expected = R1B_CLEANED[name] + assert _redact(text) == expected + assert _redact(expected) == expected + + +@pytest.mark.parametrize("text", R1B_KEPT, ids=range(len(R1B_KEPT))) +def test_rule_1b_leaves_these_to_th9491(text: str) -> None: + assert _redact(text) == _base(text) + + +@pytest.mark.parametrize("text", list(R1B_LIMITS), ids=range(len(R1B_LIMITS))) +def test_rule_1bs_named_limits_are_pinned(text: str) -> None: + assert _redact(text) == R1B_LIMITS[text] + + +def test_rule_1bs_bounds_hold_at_each_edge() -> None: + """The bounds (L3): 64 before the first : or @, 64 from the last @ to the /, 1-64 from the + / to the path's @, and 129 up to the last @.""" + tail = "@db.example.com/app" + assert ( + _redact("postgres://" + "a" * 64 + ":P@ss/Xk9" + tail) + == "postgres://db.example.com/app" + ) + assert "Xk9" in _redact("postgres://" + "a" * 65 + ":P@ss/Xk9" + tail) + assert ( + _redact("postgres://admin:P@" + "s" * 64 + "/Xk9" + tail) + == "postgres://db.example.com/app" + ) + assert "Xk9" in _redact("postgres://admin:P@" + "s" * 65 + "/Xk9" + tail) + assert ( + _redact("postgres://admin:P@ss/" + "x" * 64 + tail) + == "postgres://db.example.com/app" + ) + # Before the authority's last @: 129 characters cleans, 130 doesn't. + for before, cleaned in ((129, True), (130, False)): + user = "a:P@" + "q" * (before - 4) + "@" + out = _redact("postgres://" + user + "s/Xk9" + tail) + assert (out == "postgres://db.example.com/app") is cleaned, before + + +def test_rule1_returns_one_sorted_list_with_overlaps_merged() -> None: + """M4: `remove` needs spans in order and apart.""" + text = "a://u:x://v:p@h/x@host.example" + assert secret_shapes.rule1(text) == [(4, 18)] + assert secret_shapes.remove(text, secret_shapes.rule1(text)) == "a://host.example" + + +# -- Change review r2 (cr2) ---------------------------------------------------------------------- + + +def test_a_one_url_value_cut_at_600_masks_its_cut_key() -> None: + """cr2 M5: the join-first branch keeps the raw value's 600-character test, so a key cut short + at the end is masked to it, joined or not.""" + for between in ("/tokens/", "\n/tokens/"): + value = _SHOWN("https://h.example/" + "a" * 547 + between + _GH) + assert len(value) == 600 + + out = _redact(value) + + assert out.endswith("/tokens/[redacted]..."), between + assert "ghp_A1b2" not in out + + +@pytest.mark.parametrize( + "text, expected", + [ + ("password:2024/Zq8wYv3Lp9 ok", "password:[redacted] ok"), + ("see token:8080/x9 now", "see token:[redacted] now"), + ], +) +def test_a_name_with_a_port_like_value_outside_a_host_is_masked( + text: str, expected: str +) -> None: + """cr2 M5: only a name in a URL's host is skipped for its port (cr1 M1); outside a host, a + value that begins with digits and a / is still a value.""" + assert _redact(text) == expected + + +def test_a_one_url_value_whose_password_held_a_space_is_joined_whole() -> None: + """cr2 L1, a named limit: once rule 1 removes a password broken by a space, the value is one + URL, and TH-9491's join takes the next line into it. No secret goes; the line is lost.""" + assert _redact("redis://:pa ss@h:6379\nREDIS_DB=0") == "redis://h" + + +@pytest.mark.parametrize( + "text, expected", + [ + # `_URL_END` stops at the next ://, so the URL after a span in a host is kept. + ( + "see x://password=Zq8w1,b/https://h.example/docs now", + "see [redacted]https://h.example/docs now", + ), + # Rule 1b's host: localhost with a trailing dot, and with a port rule 1 doesn't read. + ( + "postgres://admin:P@ss/Xk9@localhost.:5432/app", + "postgres://localhost.:5432/app", + ), + ( + "postgres://admin:P@ss/Xk9@localhost:{{port}}/app", + "postgres://localhost/app", + ), + ], +) +def test_the_url_end_and_rule_1bs_localhost_host(text: str, expected: str) -> None: + """cr2 L5.""" + assert _redact(text) == expected + + +def test_a_url_query_before_a_key_body_doesnt_defeat_the_cut_rules() -> None: + """r1 M1: Terraform state cut at 600 with a query URL before private_key_pem. Both + 600-character tests are made on the raw value, which step 2 would shorten.""" + state = _SHOWN( + '{"attributes": {"endpoint": "https://sts.amazonaws.com/?Action=AssumeRole&Version=2011-' + '06-15", "private_key_pem": "-----BEGIN RSA PRIVATE KEY-----\\n' + + _PEM_JSON + + '\\n-----END RSA PRIVATE KEY-----\\n"}}' + ) + assert len(state) == 600 + + out = _redact(state) + + assert "MIIEowIBAAKC" not in out and "FAKEfake0123" not in out + assert out.endswith('"private_key_pem": "[redacted]') + # A key cut short at 600 is masked to the end, behind a query too. + shown = _SHOWN("see https://h.example/x?a=1 then " + "y " * 270 + _GH) + assert _redact(shown).endswith(" [redacted]...") + + +def test_the_page_outcomes_hold() -> None: + """The page's own outcomes, as round 3 listed them: None means TH-9491's result stands.""" + for text, expected in PAGE_OUTCOMES.items(): + assert _redact(text) == (_base(text) if expected is None else expected), text + + +# -- S3 and S4: the corpus ---------------------------------------------------------------------- + + +def _corpus() -> dict[str, str]: + cases = {} + for label, group in ( + ("r1", R1_CASES), + ("r1 extra", R1_EXTRA), + ("r2", R2_CASES), + ("r3", R3_CASES), + ("r4 honest", R4_HONEST), + ): + cases.update({f"{label}: {name}": text for name, text in group.items()}) + cases.update({f"r4 secret: {name}": text for name, (text, _) in R4_SECRETS.items()}) + cases.update( + {f"r4 drift {i}: {name}": text for i, (name, text) in enumerate(R4_DRIFT)} + ) + return cases + + +CORPUS = _corpus() +# Pieces of the fake secrets in the corpus. None may survive cleaning, outside the limits below. +SECRET_PARTS = ( + "A1b2C3d4E5f6G7h8I9j0", + "FAKEfake0123456789", + "FAKEFAKE12345678", + "FAKEfakeFAKEfakeFAKEfake", + "AAFAKEfake", + "FAKEsigFAKEsig", + "MIIEFAKE", + "ABCDEFFAKE", + "ABCDFAKE", + "MHcCA", + "hunter2", + "ter2", + "Zq8wYv3Lp9", + "s3cr3tvalue99", + "wJalrXUtnFEMI", + "MIIEowIBAAKC", + "MIIEvQIBADAN", + "b3BlbnNzaC1rZXkt", + "Acme.Admin1", + "x7.Kq9vL2mPz", + "ya29.a0Af", + "FAKEazureFAKEkey", + "battery", + "correct horse", + "sk-1", + "pa/ss", + "pa://ss", + "q8#w(v3l)p9", + "FwoGZXIv", + "XXXXXXXXXXXXXXXX", + "abcdef0123456789abcdef", + "A0FAKE0FAKE", + "8812", + "ghp_A1b2C3d4", +) +# Cases where a secret is kept, each a limit the design page names. +SECRETS_KEPT = { + "r1: page: ?# in password": "a password that starts with digits", + "r2: r1 digits slash": "a password that starts with digits", + "r2: nv ": "=> isn't matched (r3 H3)", + "r3: N go :=": ":= isn't matched (r3 H3)", + "r3: N cli flag": "--password X isn't matched (r3 H3)", + "r3: N bearer": "Authorization: Bearer X isn't matched (r3 H3)", + "r3: N yaml phrase": "an unquoted passphrase with spaces (r4 L6)", + "r4 secret: yaml block scalar": "a YAML block scalar (r4 L6)", + "r4 secret: unquoted passphrase": "an unquoted passphrase with spaces (r4 L6)", + "r4 secret: rule1 blank line": "a blank line inside a password (r4 L3)", + "r4 secret: camel passwd/pwd": "dbPwd isn't a name (r4 L6)", + "r4 secret: pass camel": "smtpPass, a best-effort form (TH-9555)", + "r4 drift 15: connection_string": "Pwd= isn't a name", + "r4 drift 11: mailchimp_api_key": "Mailchimp's shape is left out on purpose", + "r4 drift 12: discord_token": "the bare Discord bot token is a named limit (r4 L5)", + "r2: cut ghp": "a key cut short in a text that isn't the 600-character cut (r4 L1)", + "r2: cut sk-proj end": "shorter than the key's least length, and not the cut (r4 L1)", + "r4 honest: pagination nextToken": "a pagination name's value is honest (r4 L2)", +} +# Text with no secret that the rules change, each as the design page says they would. +HONEST_CHANGED = { + "r2: H a.com:x y@z": "rule 1 reads 'x y' as a password broken by a space", + "r2: nv placeholder": "a placeholder 16 or more characters long (page, what can go wrong)", + "r2: nv v2beta": "token: v2beta is masked (page, what can go wrong)", + "r2: H pem prose": "the bare header is masked; the instruction after it is kept", + "r3: H changeme123": "a letter and a digit", + "r3: H v2beta": "token: v2beta is masked (page, what can go wrong)", + "r3: H ident digit": "an identifier with a digit isn't code", + "r3: H #channel2": "a markdown anchor after an address loses its values (r3 L6)", + "r3: H pem prose": "the bare header is masked; the instruction after it is kept", + "r4 honest: pem prose": "the bare header is masked; the instruction after it is kept", + "r4 honest: pagination next_page_token": "a best-effort form (TH-9555)", + "r4 honest: pagination nextForwardToken": "a best-effort form (TH-9555)", +} + + +def test_every_secret_in_the_corpus_is_masked() -> None: + leaked = { + name: [part for part in SECRET_PARTS if part in text and part in _redact(text)] + for name, text in CORPUS.items() + if name not in SECRETS_KEPT + } + assert {name: parts for name, parts in leaked.items() if parts} == {} + + +def test_honest_text_in_the_corpus_comes_out_unchanged() -> None: + """IDs, hashes, base64, code, markdown and pagination names: as TH-9491 leaves them.""" + changed = [ + name + for name, text in CORPUS.items() + if not any(part in text for part in SECRET_PARTS) + and name not in HONEST_CHANGED + and _redact(text) != _base(text) + ] + assert changed == [] + for name in HONEST_CHANGED: + assert _redact(CORPUS[name]) != _base(CORPUS[name]), name + + +def test_cleaning_the_corpus_twice_changes_nothing_and_logs_once( + caplog: pytest.LogCaptureFixture, +) -> None: + """S4: idempotence, and one log line per call, none on the second pass.""" + for name, text in CORPUS.items(): + caplog.clear() + with caplog.at_level(logging.DEBUG, logger="fi.alk.providers.agent"): + once = _redact(text) + first = _logged(caplog) + caplog.clear() + twice = _redact(once) + second = _logged(caplog) + assert twice == once, name + assert len(first) <= 1 and second == [], name + + +# -- S5 and S5b: speed -------------------------------------------------------------------------- + +_SETUP = r""" +import logging, random, string, time +logging.disable(logging.WARNING) +from fi.alk.providers.agent import _redact +GH = "ghp_" + "A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8" +def rep(unit): + return (unit * (M // len(unit) + 1))[:M] +def dense(): + pick, unreserved, out, size = random.Random(9), string.ascii_letters + string.digits + "-._~", [], 0 + while size < M: + host = "".join(pick.choice(string.ascii_lowercase) for _ in range(8)) + key = "".join(pick.choice(unreserved) for _ in range(10)) + sentence = f"Call https://u{size}:p{key}@{host}.example/v{size % 97}?key={key} then reply. " + out.append(sentence) + size += len(sentence) + return "".join(out) +""" + +# The budget for each 5 MiB shape below, best of 3 (design "Speed", and TH-9553's four shapes). +# Set by the Coordinator for 3.12.3 under load (change review r1, cr1 M2). +_S5_BUDGET_S = 1.5 +_MIB = 1024 * 1024 +_SHAPES = { + "prose with no key": 'rep("The quick brown fox jumps over the lazy dog. ")', + "prose with names and no key": 'rep("Each token costs; the secret is batching, the password policy is long. ")', + "TH-9491 address-dense": "dense()", + "eyJ repeated": 'rep("eyJ")', + "digits then :AA": 'rep("1234567:AA")', + "an unterminated key block": '"-----BEGIN RSA PRIVATE KEY-----\\n" + rep("A" * 64 + "\\n")', + "-----BEGIN headers with no END": 'rep("-----BEGIN RSA PRIVATE KEY-----")', + "overlapping named values": 'rep("password=")', + "TH-9553 URLs with a key in each host": 'rep("https://" + GH + ".example.com/x ")', + "TH-9553 a one-URL value with a name on every line": '"https://h.example/x" + rep("\\npassword=Zq8w1")', + "TH-9553 token: abc repeated": 'rep("token: abc ")', +} + + +# cr1 H1 (change review r1): a name in a URL's host, repeated with no whitespace, and one such +# span before a long run of scheme characters. Each took over 10 s at 1 MiB or less when +# _url_around searched forward from every span. 1 MiB, best of 3, under 1 s: pinned for +# linearity, not timed at 5 MiB (the Coordinator). +_H1_BUDGET_S = 1.0 +_H1_SHAPES = { + "a name in a host, repeated": 'rep("a://password=Zq8w1,b/")', + "one host span, then a run, then /y://z": '"a://password=Zq8w1,b/" + "x" * (M - 27) + "/y://z"', +} + + +def _best_of_three(shape: str, expression: str, size: int) -> float: + code = ( + _SETUP + + f"M = {size}\n" + + f"value = {expression}\n" + + "best = 9.0\n" + + "for _ in range(3):\n" + + " start = time.perf_counter()\n" + + " _redact(value)\n" + + " best = min(best, time.perf_counter() - start)\n" + + "print(best)\n" + ) + try: + done = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + timeout=60, + check=True, + ) + except subprocess.TimeoutExpired: + pytest.fail(f"{shape}: three cleans took more than 60 s") + return float(done.stdout.strip()) + + +@pytest.mark.parametrize("shape", list(_SHAPES)) +def test_five_mib_cleans_within_the_budget(shape: str) -> None: + best = _best_of_three(shape, _SHAPES[shape], 5 * _MIB) + assert best < _S5_BUDGET_S, f"{shape}: {best:.3f} s" + + +@pytest.mark.parametrize("shape", list(_H1_SHAPES)) +def test_a_name_in_a_url_host_cleans_in_linear_time(shape: str) -> None: + best = _best_of_three(shape, _H1_SHAPES[shape], _MIB) + assert best < _H1_BUDGET_S, f"{shape}: {best:.3f} s" + + +# Rule 1b's shapes and rule 1's, 5 MiB each (design addendum v2.5.2, "Speed"; the Coordinator's +# call): the cleaner with 1b against the same build without it, `secret_shapes.rule1b` swapped +# out in the same process, best of 5 each, interleaved. With 1b, at most 2 times without it, +# and under 5 s. A shape with no @ runs no 1b match, so it can't fail on the ratio: the 5 s +# ceiling is what pins it, as the Coordinator's call has it (cr2 L6). +_RULE1B_RATIO = 2.0 +_RULE1B_CEILING_S = 5.0 +_RULE1_SHAPES = { + "rule 1's scheme://u: with no @": 'rep("https://u:")', + # One @ at the very end: rule 1 must find it once, not again from every scheme://. + "rule 1's scheme://u: then one @": 'rep("https://u:")[:-1] + "@"', + "TH-9553 https://u:p repeated": 'rep("https://u:p ")', + "rule 1's u:p@h/x@, no host": 'rep("https://u:p@h/x@ ")', + "rule 1's u:p@h/x@h.io": 'rep("https://u:p@h/x@h.io ")', + # Owl's reviewer S's and Hawk's reviewer T's shapes for 1b. + "1b a://u:": 'rep("a://u:")', + "1b a://u:p@h/x@": 'rep("a://u:p@h/x@")', + "1b a://:@/@": 'rep("a://:@/@")', + "1b a://u@": 'rep("a://u@")', + "1b a:// then x: 40 times": 'rep("a://" + "x:" * 40)', + "1b a:// then x@ 40 times": 'rep("a://" + "x@" * 40)', + "1b each part at its bound, no host": 'rep("a://" + "u" * 30 + ":" + "p" * 29 + "@"' + ' + "q" * 60 + "/" + "s" * 60 + "@ ")', + "1b port guard, :1 95 times": 'rep("a://u@" + ":1" * 95 + "/")', + "1b port guard, :{ 95 times": 'rep("a://u@" + ":{" * 95 + "/")', + "1b port guard, u@h:80/x@h.io": 'rep("a://u@h:80/x@h.io ")', + "1b a label run after the path's @": 'rep("a://u:p@h/x@" + "a" * 60 + "!")', + "1b a label host, one URL": 'rep("a://u:p@h/x@db/")', + "1b a dotted run after the path's @": 'rep("a://u:p@h/x@" + "a." * 30 + "!")', + "1b a hit, one URL": 'rep("a://u:p@h/x@h.io")', + "1b a hit and rule 1 together": 'rep("a://u:x://v:p@h/x@h.io ")', +} + + +def _with_and_without_1b(shape: str, expression: str) -> tuple[float, float]: + """The best of 5 without rule 1b and with it, interleaved in one process.""" + code = ( + _SETUP + + f"M = {5 * _MIB}\n" + + f"value = {expression}\n" + + "from fi.alk.providers import secret_shapes\n" + + "with_1b = secret_shapes.rule1b\n" + + "best = {False: 99.0, True: 99.0}\n" + + "for _ in range(5):\n" + + " for on in (False, True):\n" + + " secret_shapes.rule1b = with_1b if on else (lambda text: [])\n" + + " start = time.perf_counter()\n" + + " _redact(value)\n" + + " best[on] = min(best[on], time.perf_counter() - start)\n" + + "print(best[False], best[True])\n" + ) + try: + done = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + timeout=150, + check=True, + ) + except subprocess.TimeoutExpired: + pytest.fail(f"{shape}: ten cleans took more than 150 s") + without, with_1b = map(float, done.stdout.split()) + return without, with_1b + + +@pytest.mark.parametrize("shape", list(_RULE1_SHAPES)) +def test_rule_1b_costs_at_most_twice_the_cleaner_without_it(shape: str) -> None: + without, with_1b = _with_and_without_1b(shape, _RULE1_SHAPES[shape]) + assert with_1b <= _RULE1B_RATIO * without, ( + f"{shape}: {with_1b:.3f} s, {without:.3f} s without" + ) + assert with_1b < _RULE1B_CEILING_S, f"{shape}: {with_1b:.3f} s" + + +# Rule 1's three S5 shapes, 5 MiB against 1 MiB (the Coordinator's call, cr3): a slowdown that +# grows faster than the text fails here, at 5 times the size, without waiting for a timeout. +_RULE1_GROWTH = 6.0 +_RULE1_LINEAR_SHAPES = ( + "rule 1's scheme://u: with no @", + "rule 1's scheme://u: then one @", + "TH-9553 https://u:p repeated", +) + + +@pytest.mark.parametrize("shape", _RULE1_LINEAR_SHAPES) +def test_rule_1s_shapes_grow_with_the_text(shape: str) -> None: + """Best of 5 at each size, interleaved in one process: 5 MiB at most 6 times 1 MiB.""" + code = ( + _SETUP + + "sizes = (1024 * 1024, 5 * 1024 * 1024)\n" + + "values = {}\n" + + "for M in sizes:\n" + + f" values[M] = {_RULE1_SHAPES[shape]}\n" + + "best = dict.fromkeys(sizes, 99.0)\n" + + "for _ in range(5):\n" + + " for size in sizes:\n" + + " start = time.perf_counter()\n" + + " _redact(values[size])\n" + + " best[size] = min(best[size], time.perf_counter() - start)\n" + + "print(best[sizes[0]], best[sizes[1]])\n" + ) + try: + done = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + timeout=150, + check=True, + ) + except subprocess.TimeoutExpired: + pytest.fail(f"{shape}: ten cleans took more than 150 s") + one, five = map(float, done.stdout.split()) + assert five <= _RULE1_GROWTH * one, ( + f"{shape}: {five:.3f} s at 5 MiB, {one:.3f} s at 1" + ) + + +class _Spy: + """Stands in for a compiled pattern and counts every call made on it.""" + + calls = 0 + + def __init__(self, pattern: re.Pattern[str]) -> None: + self._pattern = pattern + + def __getattr__(self, name: str): + method = getattr(self._pattern, name) + if not callable(method): + return method + + def counted(*args, **kwargs): + _Spy.calls += 1 + return method(*args, **kwargs) + + return counted + + +def _spied(value): + if isinstance(value, re.Pattern): + return _Spy(value) + if isinstance(value, tuple): + return tuple(_spied(item) for item in value) + if isinstance(value, list): + return [_spied(item) for item in value] + if isinstance(value, dict): + return {key: _spied(item) for key, item in value.items()} + return value + + +def test_text_with_no_literal_and_no_name_calls_no_pattern( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """S5b (r1 M5): the str.find pre-check, not the regex engine's own literal search, is what + passes such text over.""" + for name, value in list(vars(secret_shapes).items()): + spied = _spied(value) + if spied is not value and not callable(value): + monkeypatch.setattr(secret_shapes, name, spied) + _Spy.calls = 0 + + assert ( + secret_shapes.find("The quick brown fox jumps over the lazy dog. " * 2000) == [] + ) + assert _Spy.calls == 0 + # The spies do see a call when there is something to look at. + assert secret_shapes.find("password=" + _PW) + assert _Spy.calls > 0 From c62d03cac07a974cdfd07f9ac2397e27888daf31 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Thu, 12 Nov 2026 10:55:19 +0530 Subject: [PATCH 2/2] ci(harness): run test_outbound.py and test_contract_writer.py in sdk-smoke Both files are touched by this change and pass whole on the base (c0e51b7e: 235 and 43 passed on Python 3.12.3). Refs: TH-9500 Signed-off-by: khushalsonawat --- .github/workflows/sdk-smoke.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/sdk-smoke.yml b/.github/workflows/sdk-smoke.yml index 8589e0e0d..8a330beb9 100644 --- a/.github/workflows/sdk-smoke.yml +++ b/.github/workflows/sdk-smoke.yml @@ -90,6 +90,8 @@ jobs: tests/harness/test_process_preflight.py tests/providers tests/harness/test_scenario_source_provision_agent.py + tests/harness/test_outbound.py + tests/harness/test_contract_writer.py tests/harness/test_framework_detection.py tests/harness/test_hosted_entrypoint.py tests/harness/test_hosted_entrypoint_provision_retry.py