fix(harness): catch secrets by shape in the kit's one cleaner (TH-9500)

.github/workflows/sdk-smoke.yml

@@ -90,6 +90,8 @@ jobs:
           tests/harness/test_process_preflight.py
           tests/providers
           tests/harness/test_scenario_source_provision_agent.py
+          tests/harness/test_outbound.py
+          tests/harness/test_contract_writer.py
           tests/harness/test_framework_detection.py
           tests/harness/test_hosted_entrypoint.py
           tests/harness/test_hosted_entrypoint_provision_retry.py

src/fi/alk/harness/hosted_chat_entrypoint.py

@@ -39,6 +39,7 @@
 )
 from .tools import schema
 from .session import ARTIFACT, DONE, RESULT, STAGE_IDLE_TIMEOUT_SECONDS, TEXT, TOOL, Event, Stage
+from ..providers.secret_shapes import ChatHold
 from ..providers.agent import (
     _LETTER,
     _QUERY_OR_FRAGMENT,
@@ -167,6 +168,11 @@
 # cleaned token by token, split at every whitespace; any whitespace is then a cut, and each
 # piece is cleaned the prose way too, so it splits at the same tokens and keeps its whitespace.
 #
+# The secret shapes cross whitespace, though: a password broken by a space or a line break, a
+# query on the next piece, a named value and its value, and a key block. So the cut is narrowed by
+# ChatHold (providers/secret_shapes.py): any whitespace is a cut only where none of those is
+# still open or has just closed, and the shed below asks it where it may split (TH-9500).
+#
 # The holdback is bounded. Past _HOLD_SOFT characters with no cut, what is held is shed:
 # - with a ? or # after its first URL start, the rest of the token is that URL's query. The held
 #   text is sent cleaned (the URL with its host and path), and the rest of the token is dropped
@@ -191,6 +197,13 @@
 class _DeltaCleaner:
     """The cleaned text of one streamed message, let out piece by piece."""
 
+    # A run of the breaks the one-URL join removes (cr3 H1, `_line_cut`).
+    _BREAK_RUN = re.compile(r"[\t\r\n]+")
+    # Where a URL's authority ends, and so its userinfo (`_line_cut`).
+    _AUTHORITY_END = re.compile(r"[/?#]")
+    # A scheme's characters, any of them first (`_learn_past`, cr3 M6).
+    _RUN_CHARS = re.compile(r"[A-Za-z0-9+.\-]*")
+
     def __init__(self) -> None:
         self.held = ""
         self.dropping = False
@@ -202,6 +215,14 @@ def __init__(self) -> None:
         # Before prose is known: a space has come after the opening began, with only whitespace
         # after it so far.
         self._spaced = False
+        # Past the soft bound, while the opening's run may still end in something other than
+        # "://" (`_learn_past`).
+        self._past: str | None = None
+        # Before prose: the joined index of a cut `_line_cut` held at 0, while the held text has
+        # only grown since (cr4 M2).
+        self._held_cut: int | None = None
+        # Where a cut may fall without splitting a secret's match (TH-9500).
+        self._hold = ChatHold()
 
     def feed(self, text: str) -> str:
         out: list[str] = []
@@ -223,19 +244,21 @@ def feed(self, text: str) -> str:
                 return "".join(out)
             text = text[cut.start() :]
             self.dropping = False
-        last = _ANY_CUT.search(text[::-1]) if self.prose else None
-        if last is None:
-            self.held += text
-        else:
-            end = len(text) - last.start()
-            out.append(self._clean(self.held + text[:end]))
-            self.held = text[end:]
+        # TH-9500: the seam cuts where this would, at the last whitespace once the text is
+        # prose, but never into a match that is still open or has just closed: rule 1, rule 2, a
+        # named value and a key block all cross whitespace.
+        self._hold.prose = self.prose
+        end = self._hold.cut(self.held, text)
+        whole = self.held + text
+        if end:
+            out.append(self._clean(whole[:end]))
+        self.held = whole[end:]
         if len(self.held) > _HOLD_SOFT:
             out.append(self._shed())
         return "".join(out)
 
     def flush(self) -> str:
-        held, self.held, self.dropping = self.held, "", False
+        held, self.held, self.dropping, self._held_cut = self.held, "", False, None
         return self._clean(held) if held else ""
 
     def _learn(self, text: str) -> None:
@@ -252,51 +275,196 @@ def _learn(self, text: str) -> None:
                 return
             self._spaced = True
         if len(self._opening) >= _HOLD_SOFT:
+            self._learn_past(text)
             return
-        self._opening += text[: _HOLD_SOFT - len(self._opening)]
+        taken = _HOLD_SOFT - len(self._opening)
+        self._opening += text[:taken]
         run = _SCHEME_RUN.match(self._opening)
         rest = self._opening[run.end() :] if run else self._opening
         # Not a letter first, or a scheme's run followed by anything but "://".
         if run is None or (rest and not "://".startswith(rest[:3])):
             self.prose = True
+        elif len(self._opening) >= _HOLD_SOFT:
+            # The opening's run, or its "://", isn't over at the soft bound: what follows still
+            # decides, as cleaning the whole text decides it (cr3 M6). Held as one URL, a run
+            # that ends in a line break would have the URL lines after it glued onto it, and
+            # their queries read as its path.
+            self._past = rest
+            self._learn_past(text[taken:])
+
+    def _learn_past(self, text: str) -> None:
+        """Past the soft bound: the opening's run, then "://" or not (`_past` holds what
+        followed the run so far; "" while the run goes on, None once decided)."""
+        past = self._past
+        if past is None or not text:
+            return
+        if not past:
+            end = self._RUN_CHARS.match(text).end()
+            if end == len(text):
+                return
+            text = text[end:]
+        past = (past + text)[:3]
+        if not "://".startswith(past):
+            self.prose = True
+        self._past = None if self.prose or past == "://" else past
 
     def _clean(self, piece: str) -> str:
         if not self.prose:
             # Joined as cleaning the whole joins a message that is one URL. Whitespace at the end
             # is after the message's last character (no part of the stripped reply), so it is
             # kept as it came.
             body = piece.rstrip()
-            return _redact(body.translate(_URL_BREAKS)) + piece[len(body) :]
+            return self._joined_clean(body) + piece[len(body) :]
         # The space in front only keeps the cleaner from reading the piece as one URL, which
         # would drop its tabs and line breaks; the prose way keeps them, and the space comes off.
         return _redact(" " + piece)[1:]
 
+    def _line_cut(self, raw: str, joined: str, at: int) -> int:
+        """Before prose, where in the held text a cut the branches put at `at` in its joined form
+        falls: at the start of the break run before the line that holds it (cr3 H1). 0 when
+        nothing may go yet.
+
+        The branches find URL starts on the joined text, where a line is glued onto the next
+        line's scheme (`DB_PASSWORD=V` then `https://…` reads `Vhttps://`). Cut there, the name
+        would go out alone and its value be cleaned as a scheme, lower-cased. Cleaning the whole
+        text masks line starts before the join, so a cut keeps lines whole, and each piece sent is
+        masked and joined as the whole is.
+
+        Nothing is sent while 0 holds, so the held text only grows; the answer for the same `at`
+        stays 0 until a later URL start moves `at`. It is kept (`_held_cut`), so a long hold
+        isn't walked again on every delta (cr4 M2)."""
+        if at == self._held_cut:
+            return 0
+        # The break runs, found once: the walk goes back over them by index, so a line costs
+        # its own length however far the held text reaches back.
+        runs = [(found.start(), found.end()) for found in self._BREAK_RUN.finditer(raw)]
+        end, line_at, i = len(raw), len(joined), len(runs)
+        while True:
+            # Back over the lines from the end, to the one that holds `at`.
+            i -= 1
+            if i < 0:
+                self._held_cut = at
+                return 0
+            run, brk = runs[i]
+            line_at -= end - brk
+            if at >= line_at:
+                break
+            end = run
+        # A name that begins the line before, with its value on this line (`token:` then `V`), is
+        # one match: the two lines stay together. Such a value is a span found with the line
+        # before and not without it.
+        if i:
+            from ..providers.secret_shapes import line_starts
+
+            before = runs[i - 1][1]
+            stop = runs[i + 1][0] if i + 1 < len(runs) else len(raw)
+            own = {span[0] + brk - 1 for span in line_starts(raw[brk - 1 : stop])}
+            with_before = line_starts(raw[before - 1 : stop])
+            if any(
+                span[0] + before - 1 >= brk and span[0] + before - 1 not in own
+                for span in with_before
+            ):
+                line_at -= run - before
+                run = runs[i - 1][0]
+        # A URL before the cut keeps its userinfo with its host: the text after the cut, up to
+        # the next ://, may still be that userinfo (`postgres://admin:Xk9/Qp+Zr`, then `8@db…`,
+        # a rule 1b password holding a /). So while an @ comes before that :// (the last URL's
+        # own, which the cut is at or before), nothing goes. Nor does it while the text before
+        # the cut ends in a userinfo still open: an @ after a : since the last ://, with no /,
+        # ? or # between that @ and the cut (`…:P@ss/Xk9@`, then the host). The last :// is
+        # looked for in the joined text, where a scheme broken by a line break is whole again.
+        scheme = joined.rfind("://", 0, line_at)
+        # A cut at the start of a line that is only a URL's own `://` (its scheme on the line
+        # before) would send the scheme alone and leave the userinfo to be cleaned without it,
+        # so nothing goes. A known limit: an honest reply with such a line and then more than
+        # about 12 K of glued URLs reaches the hard bound and streams `[redacted]`.
+        inside = joined.rfind("://", 0, line_at + 3)
+        if inside >= 0 and line_at - 3 < inside <= line_at:
+            self._held_cut = at
+            return 0
+        if scheme >= 0:
+            following = joined.find("://", line_at)
+            sign = joined.rfind("@", scheme, line_at)
+            if "@" in joined[line_at:following] or (
+                sign > joined.find(":", scheme + 3, sign) >= 0
+                and not self._AUTHORITY_END.search(joined, sign, line_at)
+            ):
+                self._held_cut = at
+                return 0
+        return run
+
+    def _joined_clean(self, text: str) -> str:
+        """Held text before prose, cleaned: masked at its line starts, then joined, as the whole
+        text is (cr2 H1). The kinds masked go on this clean's one log line."""
+        from ..providers.agent import _line_starts_joined
+
+        found: list[str] = []
+        joined = _line_starts_joined(text, found)
+        return _redact(joined, hits=found)
+
     def _shed(self) -> str:
-        held = self.held
+        # 1. The limit: feed calls this once more than _HOLD_SOFT characters are held.
+        # 2. The split, from the seam (TH-9500): None when nothing it holds is open, so the
+        #    branches below run on all of the held text; otherwise the start of the held text's
+        #    first open match that may not be cut: a key block, or a key, whose hold starts at
+        #    its URL piece when it is in one.
+        split = self._hold.shed_cut(self.held)
+        # 6. A limit of 0: the held text opens with such a match, so nothing goes out yet.
+        if split == 0:
+            return ""
+        rest = ""
+        if split is not None:
+            self.held, rest = self.held[:split], self.held[split:]
+        # 3. The branches, run on the text before the split. Before prose they run on the
+        #    joined text, and a cut keeps lines whole (`_line_cut`, cr3 H1).
+        held = head = raw = self.held
         if not self.prose:
             if self._spaced:
-                self.held = ""
+                self.held, self._held_cut = "", None
                 return self._clean(held)
-            self.held = held = held.translate(_URL_BREAKS)
+            held = raw.translate(_URL_BREAKS)
         starts = []
-        for found in _START.finditer(held):
-            letter = _LETTER.search(held, found.start(), found.end() - 3)
-            starts.append(letter.start() if letter else found.end() - 3)
+        for start in _START.finditer(held):
+            letter = _LETTER.search(held, start.start(), start.end() - 3)
+            starts.append(letter.start() if letter else start.end() - 3)
+
+        def cut(at: int) -> tuple[str, str]:
+            # The held text before `at` (an index into `held`), cleaned, and the rest, held again.
+            if self.prose:
+                return _redact(held[:at]), held[at:]
+            if len(raw) != len(held):
+                at = self._line_cut(raw, held, at)
+            if not at:
+                return "", raw
+            # Text goes, so the held text's indexes move: a held cut no longer stands.
+            self._held_cut = None
+            return self._joined_clean(raw[:at]), raw[at:]
+
         if not starts:
             tail = _SCHEME_TAIL.search(held)
-            keep = tail.start() if tail else len(held)
-            self.held = held[keep:]
-            return _redact(held[:keep])
-        if _QUERY_OR_FRAGMENT.search(held, starts[0]):
-            self.held, self.dropping = "", True
-            return _redact(held)
-        out = ""
-        if starts[-1] > 0:
-            out = _redact(held[: starts[-1]])
-            self.held = held = held[starts[-1] :]
-        if len(held) > _HOLD_HARD:
-            self.held, self.dropping = "", True
-            out += _REDACTED
+            out, self.held = cut(tail.start() if tail else len(held))
+        elif _QUERY_OR_FRAGMENT.search(held, starts[0]):
+            out = _redact(held) if self.prose else self._joined_clean(raw)
+            self.held, self.dropping, self._held_cut = "", True, None
+        else:
+            out, self.held = "", raw
+            if starts[-1] > 0:
+                out, self.held = cut(starts[-1])
+            # The held text itself is counted, breaks too: the join drops them, but they are held,
+            # and the walk over them (`_line_cut`) is bounded by this.
+            if len(self.held) > _HOLD_HARD:
+                self.held, self.dropping, self._held_cut = "", True, None
+                out += _REDACTED
+        # 5. The drop rule. A key inside a held URL piece is never a cut point: the split above
+        #    is that piece's start or nothing. A split is a piece's start, so the token a branch
+        #    dropped ends before it, and no fragment of it is kept. When the text the branches
+        #    ran on ends in whitespace its last token is complete, so nothing is left to drop
+        #    and the next honest word is kept (r1 L6). Before prose a tab or a line break is no
+        #    token's end: the one-URL join removes it.
+        if self.prose and self.dropping and head[-1:].isspace():
+            self.dropping = False
+        # 4. The tail from the split is held again.
+        self.held += rest
         return out
 
 

src/fi/alk/harness/outbound.py

@@ -70,6 +70,7 @@
 )
 from fi.simulate.results.futureagi import MAX_TARGET_LATENCY_TURNS
 
+from ..providers.agent import _log_redacted, _redact_text
 from .job import FailureDomain, HarnessStage
 
 logger = logging.getLogger(__name__)
@@ -277,28 +278,37 @@ def _mask_userinfo(match: re.Match[str]) -> str:
 def redact_outbound_text(value: str, extra_secret_values: tuple[str, ...] = ()) -> str:
     """Scrubs a single free-text field before it can leave the sandbox on any of the three
     channels (outbound-channels.md v1.3 "Redaction (enforced before emit)"; hosted-execution-
-    seams.md v1.11 §3 "any outbound projection ... redacts userinfo"). Two things, applied in
-    order:
-
-    1. URL userinfo (`_USERINFO_PATTERN`/`_mask_userinfo`, above) -- a password is masked and the
-       username kept (matching the contract's own `postgresql://harness:***@...` example); a bare
-       token/username-only userinfo (no `:`) is masked outright, since that shape is how a bearer
-       token appears, not a username.
-    2. `extra_secret_values` -- exact-substring replacement for a caller-supplied list of secret
-       values. Always `()` today: the adapter (P10) is what will know the job's declared secrets
-       and pass them in -- this parameter exists now so `build_event_record`/`build_result_receipt`
-       never need to change shape when that wiring lands.
-
-    NOT a general secret-content scanner -- the contract's "same secret-content scan as the
-    artifact sealer" is a separate, sealer-side obligation. This is the narrow subset this module
-    can enforce on every string field it controls without false-positiving on ordinary diagnostic
-    text.
+    seams.md v1.11 §3 "any outbound projection ... redacts userinfo"). Three things, applied in
+    order (TH-9500):
+
+    1. `extra_secret_values` -- exact-substring replacement for a caller-supplied list of secret
+       values. First, so a declared secret is matched whole before the cleaner below can mask
+       part of it and leave the rest unmatched. Always `()` today: the adapter (P10) is what will
+       know the job's declared secrets and pass them in -- this parameter exists now so
+       `build_event_record`/`build_result_receipt` never need to change shape when that wiring
+       lands.
+    2. The kit's one cleaner, `providers.agent._redact_text`, all three of its steps: a URL loses
+       its userinfo, query and fragment, and keys, key blocks and named values are masked by
+       their shape. It logs nothing itself; one line names what it removed, on the cleaner's
+       own logger (not one of the loggers `MaskSecrets` filters, so the filter never re-enters),
+       and only when it changed the text.
+    3. URL userinfo (`_USERINFO_PATTERN`/`_mask_userinfo`, above), for whatever step 2 left: a
+       password is masked and the username kept; a bare token/username-only userinfo (no `:`)
+       is masked outright, since that shape is how a bearer token appears, not a username.
+
+    Step 2 removes a URL's userinfo outright, so `https://user:pw@host` now comes out as
+    `https://host`, not `https://user:***@host`.
     """
-    redacted = _USERINFO_PATTERN.sub(_mask_userinfo, value)
+    redacted = value
     for secret in extra_secret_values:
         if len(secret) >= _SHORTEST_REDACTABLE_SECRET:
             redacted = redacted.replace(secret, "***")
-    return redacted
+    refused: list[str] = []
+    hits: list[str] = []
+    cleaned = _redact_text(redacted, refused, hits)
+    if (refused or hits) and cleaned != redacted:
+        _log_redacted(refused, hits)
+    return _USERINFO_PATTERN.sub(_mask_userinfo, cleaned)
 
 
 # =================================================================================================