Release 0.6.1

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "simpleaudit"
-version = "0.6.0"
+version = "0.6.1"
 description = "Lightweight AI Safety Auditing Framework"
 readme = "README.md"
 license = {text = "MIT"}
@@ -51,7 +51,10 @@ tracing = [
 plot = ["matplotlib>=3.5.0"]
 # Run scenario packs as Inspect AI tasks and read Inspect logs back
 # (simpleaudit.integrations.inspect).
-inspect = ["inspect-ai>=0.3.283"]
+# agent-client-protocol 1.0.0 (2026-11-09) removed acp.helpers, which inspect-ai
+# 0.3.288 still imports (it only requires >=0.12.0). Drop the cap once inspect-ai
+# supports 1.x.
+inspect = ["inspect-ai>=0.3.283", "agent-client-protocol<1"]
 dev = [
     # pytest is pinned <9 because pytest-testmon 2.x is not yet compatible with
     # pytest 9 (same pin as Studio).

simpleaudit/failures.py

@@ -0,0 +1,167 @@
+"""Which role failed a scenario, and how (structured, best effort).
+
+A failed scenario keeps its free-text ``judgment.error`` and
+``judgment.error_phase`` and gains ``judgment.failure``::
+
+    {"role": "target|auditor|judge|system",          # required
+     "kind": "rate_limit|unavailable|auth|timeout|bad_response|config|unknown",
+     "message": "<same text as judgment.error>",
+     "http_status": 429, "exception": "RateLimitError",
+     "endpoint": "...", "model": "...",
+     "retryable": true, "turn": 3}
+
+Only ``role`` (and ``kind``, ``message``, ``exception``) are always present;
+the other keys are left out when unknown. The role is set at the call site,
+so no target cooperation is needed and nothing here assumes a model, a
+provider or a target type. ``kind`` comes from the HTTP status the exception
+exposes, or its type; anything else is ``unknown``. ``retryable`` is a hint
+for consumers and does not describe the engine's own retry behaviour.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import re
+from typing import Any, Dict, Optional
+
+from .utils import describe_exception, redact_url_secrets
+
+ROLES = ("target", "auditor", "judge", "system")
+KINDS = ("rate_limit", "unavailable", "auth", "timeout", "bad_response", "config", "unknown")
+_RETRYABLE = {"rate_limit", "unavailable", "timeout", "bad_response"}
+_MAX_CHAIN = 6
+
+
+def _chain(exc: BaseException):
+    seen = set()
+    cur: Optional[BaseException] = exc
+    while cur is not None and id(cur) not in seen and len(seen) < _MAX_CHAIN:
+        seen.add(id(cur))
+        yield cur
+        cur = cur.__cause__ or cur.__context__
+
+
+def _as_status(value: Any) -> Optional[int]:
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, int) and 100 <= value <= 599:
+        return value
+    return None
+
+
+def http_status_of(exc: BaseException) -> Optional[int]:
+    """HTTP status the exception (or its cause chain) exposes, else None."""
+    for item in _chain(exc):
+        for attr in ("status_code", "status", "http_status", "code"):
+            status = _as_status(getattr(item, attr, None))
+            if status is not None:
+                return status
+        response = getattr(item, "response", None)
+        if response is not None:
+            for attr in ("status_code", "status"):
+                status = _as_status(getattr(response, attr, None))
+                if status is not None:
+                    return status
+    return None
+
+
+def _kind_from_type(exc: BaseException) -> str:
+    # Local import: provider_errors is independent of this module.
+    from .provider_errors import ProviderResponseError
+
+    for item in _chain(exc):
+        if isinstance(item, (asyncio.TimeoutError, TimeoutError)):
+            return "timeout"
+        if isinstance(item, ConnectionError):
+            return "unavailable"
+        if isinstance(item, ProviderResponseError):
+            return "bad_response"
+        name = type(item).__name__.lower()
+        if "ratelimit" in name:
+            return "rate_limit"
+        if "timeout" in name:
+            return "timeout"
+        if "authentication" in name or "permissiondenied" in name:
+            return "auth"
+        if any(
+            w in name
+            for w in ("connect", "unavailable", "overloaded", "internalserver", "badgateway")
+        ):
+            return "unavailable"
+        if name in ("jsondecodeerror", "validationerror"):
+            return "bad_response"
+    return "unknown"
+
+
+def classify_kind(exc: BaseException, http_status: Optional[int]) -> str:
+    """Provider-agnostic cause from the HTTP status, else the exception type."""
+    if http_status is not None:
+        if http_status == 429:
+            return "rate_limit"
+        if http_status in (401, 403):
+            return "auth"
+        if http_status in (408, 504):
+            return "timeout"
+        if http_status >= 500:
+            return "unavailable"
+        if 400 <= http_status < 500:
+            return "config"
+    return _kind_from_type(exc)
+
+
+def _clean_endpoint(url: Any) -> Optional[str]:
+    if not url:
+        return None
+    text = str(url)
+    # Credentials riding in the URL are never kept (not even redacted).
+    text = re.sub(r"(\w+://)[^/?#@\s]*@", r"\1", text)
+    return redact_url_secrets(text)
+
+
+def _endpoint_of(exc: BaseException) -> Optional[str]:
+    for item in _chain(exc):
+        for holder in (getattr(item, "request", None), getattr(item, "response", None)):
+            url = getattr(holder, "url", None) if holder is not None else None
+            cleaned = _clean_endpoint(url)
+            if cleaned:
+                return cleaned
+    return None
+
+
+def build_failure(
+    role: str,
+    exc: BaseException,
+    *,
+    turn: Optional[int] = None,
+    endpoint: Optional[str] = None,
+    model: Optional[str] = None,
+) -> Dict[str, Any]:
+    """The ``failure`` object for ``exc`` raised by ``role``.
+
+    ``endpoint`` and ``model`` are hints from the call site, used when the
+    exception itself does not name an endpoint. ``turn`` is 1-based.
+    """
+    role = role if role in ROLES else "system"
+    status = http_status_of(exc)
+    kind = classify_kind(exc, status)
+    failure: Dict[str, Any] = {
+        "role": role,
+        "kind": kind,
+        "message": describe_exception(exc),
+        "exception": type(exc).__name__,
+    }
+    if status is not None:
+        failure["http_status"] = status
+    where = _endpoint_of(exc) or _clean_endpoint(endpoint)
+    if where:
+        failure["endpoint"] = where
+    if model:
+        failure["model"] = str(model)
+    declared = getattr(exc, "retryable", None)
+    if isinstance(declared, bool):
+        failure["retryable"] = declared
+    elif kind != "unknown":
+        failure["retryable"] = kind in _RETRYABLE
+    if turn is not None:
+        failure["turn"] = turn
+    return failure

simpleaudit/model_auditor.py

@@ -37,6 +37,7 @@
 from .judges.compose import inherited_postprocess
 from .judges.compose import SEVERITY_RESPONSE_SCHEMA
 from .judges.default import DEFAULT_JUDGE_CRITERIA, DEFAULT_JUDGE_SEVERITY_LEVELS, DEFAULT_PROBE_PROMPT
+from .failures import build_failure
 from .provider_errors import (
     EmptyResponseError,
     _content_to_text,
@@ -130,7 +131,7 @@ class ScenarioExecution:
 #: failures, or carry the ground truth and designed severity a later
 #: re-grade (reframing.rejudge) trusts.
 _FRAMEWORK_KEYS = (
-    "parse_error", "error", "error_phase", "scenario_metadata",
+    "parse_error", "error", "error_phase", "failure", "scenario_metadata",
     "designed_severity", "designed_severity_source", "judge_fallback", "asr_reference",
 )
 
@@ -1106,6 +1107,42 @@ def _log(self, message: str, name: Optional[str] = None):
                     message = f"[{name}] {message}"
                 tqdm.write(message)
 
+    def _failure_for(
+        self,
+        role: str,
+        exc: BaseException,
+        *,
+        turn: Optional[int] = None,
+        target: Any = None,
+    ) -> Dict[str, Any]:
+        """Structured ``judgment.failure`` for ``exc`` raised by ``role``.
+
+        Endpoint and model are hints from the configuration and are only used
+        when known; a custom target contributes its own ``url``/``model`` if it
+        has them.
+        """
+        endpoint = model = None
+        try:
+            if role == "auditor":
+                endpoint = self._auditor_client_config.get("base_url")
+                model = self.auditor_model
+            elif role == "judge":
+                endpoint = self._judge_client_config.get("base_url")
+                model = None if self.judge_grade is not None else self.judge_model
+            elif role == "target":
+                tgt = target if target is not None else self.target
+                endpoint = getattr(tgt, "url", None) or getattr(tgt, "base_url", None)
+                model = getattr(tgt, "model", None)
+                if endpoint is None and type(tgt).__name__ == "ModelTarget":
+                    endpoint = self._target_client_config.get("base_url")
+                if not isinstance(endpoint, str):
+                    endpoint = None
+                if not isinstance(model, str):
+                    model = None
+        except Exception:  # hints are best effort and must never mask the error
+            endpoint = model = None
+        return build_failure(role, exc, turn=turn, endpoint=endpoint, model=model)
+
     def _fire_on_turn(
         self,
         turn_index: int,
@@ -1614,6 +1651,11 @@ async def run_scenario(
         error: Optional[str] = None
         error_traceback: Optional[str] = None
         error_phase: Optional[str] = None
+        failure: Optional[Dict[str, Any]] = None
+        # Which role the call in flight belongs to, set right before each call
+        # so an exception is attributed where it is raised.
+        current_role = "system"
+        current_turn: Optional[int] = None
         try:
             validate = getattr(self.judge_postprocess, "validate_scenario", None)
             if validate is not None:
@@ -1629,16 +1671,20 @@ async def run_scenario(
                 if trace_correlation is not None:
                     _link_scenario(trace_correlation, scenario_run_id, name)
                     trace_correlation.record(f"{scenario_run_id}_t0", scenario_trace_id)
+                # The hook opens the target's session.
+                current_role = "target"
                 await start_hook(hook_context)
             started = True
             for turn in range(turns):
                 self._log(f"--- Turn {turn + 1}/{turns} ---", name=name)
+                current_turn = turn + 1
 
                 # First turn: if the scenario defines a test_prompt, send it verbatim.
                 # Subsequent turns use the auditor model for probe generation.
                 if turn == 0 and (test_prompt or getattr(target, "default_prompt", None)):
                     probe = test_prompt or target.default_prompt
                 else:
+                    current_role = "auditor"
                     # require_content checks the raw text; a reasoning auditor
                     # cut off inside <think> (or a bare "User:") is empty only
                     # after stripping. That is a model hiccup like an empty
@@ -1703,6 +1749,7 @@ async def run_scenario(
                 if trace_correlation is not None:
                     _link_scenario(trace_correlation, scenario_run_id, name)
                     trace_correlation.record(turn_id, scenario_trace_id)
+                current_role = "target"
                 target_resp = await target.send(
                     system=self.system_prompt,
                     user=probe,
@@ -1808,6 +1855,7 @@ async def run_scenario(
             error = describe_exception(exc)
             error_traceback = _short_traceback()
             error_phase = "conversation"
+            failure = self._failure_for(current_role, exc, turn=current_turn, target=target)
             self._log(f"--- Scenario FAILED mid-conversation: {name} [{error}] ---")
         finally:
             # Runs on failure and cancellation too, so a target can release its
@@ -1875,6 +1923,7 @@ async def run_scenario(
                 error = describe_exception(exc)
                 error_traceback = _short_traceback()
                 error_phase = "judge"
+                failure = self._failure_for("judge", exc)
                 self._log(f"--- Judging FAILED: {name} [{error}] ---")
 
         if error is not None:
@@ -1894,6 +1943,8 @@ async def run_scenario(
                 # complete (see reframing.rejudge); a "judge" failure leaves a
                 # complete one.
                 "error_phase": error_phase,
+                # Which role failed and how, structured (see failures.py).
+                "failure": failure,
             }
         else:
             judgment = coerce_judgment_fields(judgment)
@@ -2065,6 +2116,8 @@ async def _run_one(scenario: Dict) -> AuditResult:
                             "severity": "ERROR",
                             "error": describe_exception(exc),
                             "traceback": _short_traceback(),
+                            # Raised outside the three calls: not attributable.
+                            "failure": build_failure("system", exc),
                         },
                     )
         with tqdm(total=total_audit_steps, desc=audit_desc, disable=not self.show_progress, position=0) as pbar_audit:

simpleaudit/reframing.py

@@ -78,6 +78,7 @@
 from simpleaudit._event_loop import run_sync
 from simpleaudit.judges import get_judge
 from simpleaudit.judges.compose import inherited_postprocess
+from simpleaudit.failures import build_failure
 from simpleaudit.provider_errors import coerce_judgment_fields
 from simpleaudit.model_auditor import ModelAuditor, _keep_grading_metadata
 from simpleaudit.repeated_results import aggregate_severities
@@ -931,7 +932,11 @@ async def _grade(
             raise
         except Exception as exc:
             failed = _error_judgment(f"Judge call failed: {describe_exception(exc)}")
-            failed.update(error=describe_exception(exc), error_phase="judge")
+            failed.update(
+                error=describe_exception(exc),
+                error_phase="judge",
+                failure=build_failure("judge", exc),
+            )
             _keep_grading_metadata(failed, scenario_meta)
             return failed, 0, 0
     if fell_back and isinstance(judgment, dict):