diff --git a/capabilities/web-security/capability.yaml b/capabilities/web-security/capability.yaml index ae06ca6..8da490f 100644 --- a/capabilities/web-security/capability.yaml +++ b/capabilities/web-security/capability.yaml @@ -1,15 +1,16 @@ schema: 1 name: web-security -version: "1.10.0" +version: "1.11.0" description: > - Web application penetration testing with 80+ attack technique playbooks + Web application penetration testing with 82 attack technique playbooks covering HTTP desync/request smuggling, cache poisoning, SSRF, SSTI, DOM vulnerabilities, authentication bypasses, parser differentials, AEM/Sling exploitation, GraphQL, OAuth, and client-side attacks. Includes HTTP client tooling with OOB callbacks via webhook.site - (API-key aware) and interactsh, Caido - integration via MCP, the Python caido-sdk-client, and the caido-mode - TypeScript SDK CLI (@caido/sdk-client / caido-ts) for curl-through-Caido + (API-key aware) and interactsh, four coexisting Caido surfaces + (caido-cli server, the Python caido-sdk-client, lightweight and + full-surface MCP servers, and the caido-mode TypeScript SDK CLI on + @caido/sdk-client / caido-ts) for curl-through-Caido testing, match & replace rules, and replay handoffs; Burp proxy integration via MCP, browser automation via agent-browser, JS static analysis via jxscout, AST-based code pattern @@ -135,7 +136,9 @@ dependencies: python: - "fastmcp>=2.0" - "httpx>=0.28" - - "caido-sdk-client" + # >= 0.3.0: versioned GraphQL transport (transport/latest vs transport/v0_56). + # Older releases break against Caido >= 0.57 replay schema. See mcp/caido.py. + - "caido-sdk-client>=0.3.0" - "cryptography>=41.0" - "pydantic>=2.0" scripts: diff --git a/capabilities/web-security/docker/Dockerfile.runtime b/capabilities/web-security/docker/Dockerfile.runtime index 6b48372..7dcf7dc 100644 --- a/capabilities/web-security/docker/Dockerfile.runtime +++ b/capabilities/web-security/docker/Dockerfile.runtime @@ -26,11 +26,15 @@ # # Tools with bundled SDK/MCP integration (require a running instance # reachable by network — the client library and MCP server are included): -# - Caido — caido-sdk-client (Python) + caido-mcp-server + the -# caido-mode TypeScript SDK CLI (@caido/sdk-client / caido-ts, -# vendored under skills/caido-mode) are all wired in; set -# CAIDO_URL to a running Caido instance. The caido-mode skill's -# node_modules are installed at provision time by +# - Caido — four surfaces against one instance, all keyed off CAIDO_URL: +# (1) caido-cli headless server, pinned >= 0.57.x +# (2) caido MCP Python, caido-sdk-client >= 0.3.0 +# (3) caido-go MCP upstream Go binary, pinned + checksummed +# (4) caido-mode skill TS CLI on @caido/sdk-client 0.4.0 +# (vendored under skills/caido-mode) +# The 0.57 replay schema break is why (1) and (2) carry floors — +# see scripts/install_tools.sh and mcp/caido.py. The caido-mode +# skill's node_modules are installed at provision time by # scripts/install_tools.sh (the skill dir is mounted at runtime, # not baked into this image); tsx is pre-installed globally here # as a cold-start aid. @@ -131,7 +135,7 @@ RUN npm install -g tsx RUN pip install --no-cache-dir \ "fastmcp>=2.0" \ "httpx>=0.28" \ - "caido-sdk-client" \ + "caido-sdk-client>=0.3.0" \ "pacu" \ "ast-grep-cli" diff --git a/capabilities/web-security/mcp/caido.py b/capabilities/web-security/mcp/caido.py index 084d079..10b7247 100644 --- a/capabilities/web-security/mcp/caido.py +++ b/capabilities/web-security/mcp/caido.py @@ -3,11 +3,18 @@ # requires-python = ">=3.12" # dependencies = [ # "fastmcp>=2.0", -# "caido-sdk-client", +# "caido-sdk-client>=0.3.0", # ] # /// """Caido proxy tools — wraps the caido-sdk-client for host interaction. +Requires caido-sdk-client >= 0.3.0. Earlier releases hardcode the pre-0.57 +replay schema (they select `collection`/`activeEntry` on ReplaySession and omit +`ReplaySessionKind`), so `caido_replay_request` and `caido_replay_sessions` +fail against Caido >= 0.57 with "Unknown field collection on type +ReplaySession". 0.3.0 added the versioned transport split (transport/latest vs +transport/v0_56) and negotiates the right schema per instance. + Auth resolution order: 1. CAIDO_PAT env var → PATAuthOptions (no connect() needed) 2. ~/.caido-mcp/token.json → TokenAuthOptions + connect() for refresh @@ -27,7 +34,12 @@ from caido_sdk_client import Client from caido_sdk_client.types.finding import CreateFindingOptions -from caido_sdk_client.types.replay_session import ReplaySendOptions +from caido_sdk_client.types.network import ConnectionInfoInput +from caido_sdk_client.types.replay_session import ( + CreateReplaySessionFromRaw, + CreateReplaySessionOptions, + ReplaySendOptions, +) from caido_sdk_client.types.scope import CreateScopeOptions from fastmcp import FastMCP @@ -35,6 +47,14 @@ DEFAULT_TOKEN_PATH = Path.home() / ".caido-mcp" / "token.json" MAX_OUTPUT_CHARS = 50_000 CONNECT_TIMEOUT = 30 + +# Upper bound on `replay.send()`. On Caido >= 0.57 the SDK starts a replay task +# and then waits on a task-finished *subscription*. If the target responds +# before that subscription is established, the completion event is missed and +# the await never returns — reproducible against a fast (localhost) target. +# The send itself still succeeds server-side, so on timeout we fall back to +# reading the session's active entry instead of hanging the MCP call forever. +REPLAY_SEND_TIMEOUT = 30 _SAFE_GET_RETRIES = 1 _SAFE_GET_RETRY_DELAY = 2.0 # seconds @@ -233,6 +253,46 @@ async def caido_get_request( return "\n".join(lines) +async def _replay_result_from_session(client: Client, session_id: object) -> str: + """Recover a replay result by reading the session's newest entry. + + Used when `replay.send()` times out waiting on the task-finished + subscription. The send has still happened server-side, so the entry + carries the real request/response. + """ + lines = ["status: DONE (recovered — task subscription timed out)"] + try: + session = await client.replay.sessions.get(session_id) + if session is None: + return "status: UNKNOWN\nerror: replay session vanished after send" + conn = await session.entries().last(1).execute() + if not conn.edges: + return "status: UNKNOWN\nerror: replay session has no entries" + # Re-fetch by id: entries listed via the session carry no response body. + entry = await client.replay.entries.get(conn.edges[-1].node.id) + if entry is None: + return "status: UNKNOWN\nerror: replay entry not found" + except Exception as exc: # noqa: BLE001 - surface, never mask + return f"status: UNKNOWN\nerror: could not recover replay result: {exc}" + + lines.append(f"entry_id: {entry.id}") + request = getattr(entry, "request", None) + if request is not None: + lines.append(f"request_id: {request.id}") + # `response` hangs off the entry, not off entry.request. + response = getattr(entry, "response", None) + if response is not None: + lines.append(f"response: {response.status_code} ({response.length} bytes)") + raw = getattr(response, "raw", None) + if raw: + text = raw.decode(errors="replace") + truncated = text[:MAX_OUTPUT_CHARS] + if len(text) > MAX_OUTPUT_CHARS: + truncated += f"\n\n... [TRUNCATED: {len(text)} chars total]" + lines.append(truncated) + return "\n".join(lines) + + @mcp.tool async def caido_replay_request( raw_request: Annotated[str, "Raw HTTP request including request line"], @@ -248,22 +308,45 @@ async def caido_replay_request( return err assert client is not None - session = await client.replay.sessions.create() - result = await client.replay.send( - session.id, - ReplaySendOptions( - raw=raw_request.replace("\\r\\n", "\r\n").encode(), - host=host, - port=port if port is not None else (443 if tls else 80), - tls=tls, - ), + raw = raw_request.replace("\\r\\n", "\r\n").encode() + connection = ConnectionInfoInput( + host=host, + port=port if port is not None else (443 if tls else 80), + is_tls=tls, ) - status_str = ( - result.task_status - if isinstance(result.task_status, str) - else str(result.task_status) + # The session must be SEEDED with the request. On Caido >= 0.57 `send()` + # updates the draft of an existing entry and then starts a replay task — a + # bare `sessions.create()` yields a session with no entries, so send() + # aborts with "Replay session has no entries". Creating from raw gives the + # session its first entry. + session = await client.replay.sessions.create( + CreateReplaySessionOptions( + request_source=CreateReplaySessionFromRaw(raw=raw, connection=connection) + ) ) + # Connection details are nested under `connection` (ConnectionInfoInput) — + # they are not flat kwargs on ReplaySendOptions. + try: + result = await asyncio.wait_for( + client.replay.send( + session.id, + ReplaySendOptions(raw=raw, connection=connection), + ), + timeout=REPLAY_SEND_TIMEOUT, + ) + except TimeoutError: + # The task-finished subscription was missed (see REPLAY_SEND_TIMEOUT). + # The request itself has almost certainly been sent, so recover the + # result from the session rather than reporting a false failure. + return await _replay_result_from_session(client, session.id) + + # ReplaySendResult exposes `status` ("DONE" | "CANCELLED" | "ERROR"). It may + # arrive as a TaskStatus enum, whose str() is "TaskStatus.DONE" — unwrap to + # the bare value so output is stable across SDK versions. + status_value = getattr(result, "status", None) + status_value = getattr(status_value, "value", status_value) + status_str = status_value if isinstance(status_value, str) else str(status_value) lines = [f"status: {status_str}"] if result.error: lines.append(f"error: {result.error}") diff --git a/capabilities/web-security/scripts/install_tools.sh b/capabilities/web-security/scripts/install_tools.sh index 034d032..2c3878a 100755 --- a/capabilities/web-security/scripts/install_tools.sh +++ b/capabilities/web-security/scripts/install_tools.sh @@ -63,10 +63,17 @@ if ! command -v kr &>/dev/null; then fi # -- Caido CLI ------------------------------------------------------------- -# Downloads the latest Caido CLI binary. Auth is handled at runtime via +# Pinned Caido CLI (headless server) release. Auth is handled at runtime via # CAIDO_URL + CAIDO_PAT env vars or the device flow login. +# +# Keep this pin >= 0.57.0. The vendored caido-mode skill runs on +# @caido/sdk-client 0.4.0, which targets the 0.57 replay schema (ReplaySession +# as an interface, `kind: ReplaySessionKind!` on createReplaySession, and +# task-based sending via startReplayTask). Pinning an older server here puts +# the client and server on opposite sides of that schema break. +# tests/test_caido_mode_skill.py enforces the floor. if ! command -v caido-cli &>/dev/null; then - CAIDO_VERSION="0.45.0" + CAIDO_VERSION="0.57.1" case "$ARCH" in aarch64|arm64) CAIDO_ARCH="aarch64" ;; *) CAIDO_ARCH="x86_64" ;; diff --git a/capabilities/web-security/skills/caido-mode/SKILL.md b/capabilities/web-security/skills/caido-mode/SKILL.md index 00d07a9..5bef684 100644 --- a/capabilities/web-security/skills/caido-mode/SKILL.md +++ b/capabilities/web-security/skills/caido-mode/SKILL.md @@ -1,9 +1,41 @@ --- name: caido-mode description: "Full Caido TypeScript SDK CLI (the official @caido/sdk-client / caido-ts library). Search HTTP history with HTTPQL, test with curl proxied through Caido (caching auth in reusable static curl config files), add match & replace (tamper) rules, manage findings/scopes/filters/environments, and organize handoffs into named replay sessions and collections. Use for rich write-side Caido automation (M&R rules, replay handoff, curl-through-Caido) that the lightweight caido-sdk (Python) and caido-proxy (MCP) skills do not cover. Requires Node.js + a reachable Caido instance." -tags: [worker] +compatibility: "Requires Node.js >= 18 and a reachable Caido instance >= 0.57 (CAIDO_URL). Vendored node_modules must be installed." +metadata: + upstream: "caido/skills@41697d8 (PR #22)" + upstream-skill: caido-mode + sdk: "@caido/sdk-client 0.4.0" + role: worker --- + + # Caido Mode Skill A CLI over Caido's API, built on the official **Caido TypeScript SDK** (`@caido/sdk-client`, @@ -25,19 +57,45 @@ Every command is then `npx tsx caido-client.ts ` and outputs JSON unles also run `npx --prefix skills/caido-mode tsx skills/caido-mode/caido-client.ts ` without `cd`, but `cd skills/caido-mode` is simplest.) -## Relationship to the other Caido skills (no interference) +## Which Caido surface should I use? + +The capability ships **four** Caido surfaces against one instance. They are independent and do +not collide — pick by the job, not by habit: + +| Surface | Use it for | Skill | +|---|---|---| +| **`caido-mode`** (this skill) | curl-through-Caido testing, Match & Replace rules, replay-session/collection handoff | this file | +| **`caido-sdk`** (Python lib) | quick in-process read/replay when you're already scripting Python | `caido-sdk` | +| **`caido` MCP** (Python, 9 tools) | lightweight history search / replay / findings as tool calls | `caido-proxy` | +| **`caido-go` MCP** (Go, 66+ tools) | batch send, race windows, intercept, environments, WS streams, tamper rules | `caido-proxy` | + +Decision shortcut: + +- **Iterating on requests against a target** → this skill (curl + `-K` config). +- **One-off lookup mid-conversation** → an MCP tool call (`caido-proxy`). +- **Need a Caido feature no other surface exposes** (intercept queue, race window, WS) → + `caido-go` MCP. +- **Already inside a Python script** → `caido-sdk`. + +### Auth isolation (why these never clobber each other) + +All four resolve the same instance from `CAIDO_URL`, but they store credentials in **different +places**, deliberately: -The capability ships three independent Caido surfaces — pick one, they don't collide: +| Surface | Credential store | Env | +|---|---|---| +| `caido-mode` | `~/.claude/config/secrets.json` → `.caido.instances[]` | `CAIDO_PAT`, `CAIDO_PROXY` | +| `caido-sdk`, `caido` MCP | `~/.caido-mcp/token.json` | `CAIDO_PAT` | +| `caido-go` MCP | `~/.caido-mcp/token.json` | `CAIDO_ACCESS_TOKEN` (`CAIDO_PAT` deprecated alias) | -- **`caido-mode` (this skill)** — TypeScript SDK CLI. Best for curl-through-Caido testing, - Match & Replace rules, and replay-session/collection handoffs. -- **`caido-sdk`** — direct Python `caido-sdk-client` calls for quick read/replay in-process. -- **`caido-proxy` / `caido-go` MCP** — MCP tool surface for history search, replay, findings. +Running `setup` here never touches `~/.caido-mcp/token.json`, and `caido-mcp-server login` never +touches `secrets.json`. Set up whichever surfaces you need, in any order. -All target the **same** Caido instance via `CAIDO_URL`/`CAIDO_PAT`. This skill caches its own auth -token in `~/.claude/config/secrets.json` (under `.caido`), which is **separate** from the -`~/.caido-mcp/token.json` used by the Python SDK skill and the MCP servers — so setting up -`caido-mode` never clobbers the other skills' auth, and vice-versa. +> **Token types are not interchangeable.** The Go MCP wants the **local instance access token** +> (from the Caido GUI: devtools console → +> `JSON.parse(localStorage.CAIDO_AUTHENTICATION).accessToken`). A Caido **Cloud PAT** (prefixed +> `caido_`) authenticates the cloud dashboard API, not your local instance. Feeding a Cloud PAT to +> `CAIDO_ACCESS_TOKEN` yields `Invalid token`. ## How to operate (read this first) diff --git a/capabilities/web-security/skills/caido-proxy/SKILL.md b/capabilities/web-security/skills/caido-proxy/SKILL.md index 9d83d45..1d199fc 100644 --- a/capabilities/web-security/skills/caido-proxy/SKILL.md +++ b/capabilities/web-security/skills/caido-proxy/SKILL.md @@ -1,6 +1,7 @@ --- name: caido-proxy -description: "Caido proxy integration for HTTP history search, request replay, fuzzing results, sitemap, and security findings via MCP. Use when you need to search proxy traffic, replay requests with modifications, triage fuzzing results, or document findings in Caido." +description: "Caido proxy integration for HTTP history search, request replay, fuzzing results, sitemap, and security findings via MCP. Covers both the lightweight `caido` server and the full-surface `caido-go` server (batch send, race windows, intercept, environments, tamper rules, WebSocket streams). Use when you need to search proxy traffic, replay requests with modifications, triage fuzzing results, or document findings in Caido without writing code." +compatibility: "Requires a reachable Caido instance (CAIDO_URL) and at least one Caido MCP server connected." --- # Caido Proxy @@ -12,6 +13,39 @@ MCP integration with Caido proxy. Results load into context -- keep queries focu > round-trips and are more efficient. Use this MCP path when the SDK is not > importable (its usual state outside the MCP's own env) or Caido is unreachable. +## Two MCP servers, one instance + +The capability wires **both** Caido MCP servers; they target the same instance +via `CAIDO_URL` and can be used interchangeably or together. + +| Server | Tools | Reach for it when | +|---|---|---| +| **`caido`** (Python) | 9 | history search, request detail, replay, scopes, findings | +| **`caido-go`** (Go) | 66+ | anything the lightweight one lacks: batch send, race windows, intercept queue, environments, filter presets, tamper (M&R) rules, WebSocket streams, sitemap, projects, workflows | + +If a tool you want isn't in your schema, it's almost certainly on `caido-go`. +Tool names differ by prefix: the Python server exposes `caido_health`, +`caido_search_requests`, `caido_get_request`, `caido_replay_request`, +`caido_list_scopes`, `caido_create_scope`, `caido_list_findings`, +`caido_create_finding`, `caido_replay_sessions`. The Go server namespaces +everything as `caido_*` too (e.g. `caido_batch_send`, `caido_race_window_send`, +`caido_list_tamper_rules`) — check your live tool schema rather than assuming. + +### Sibling non-MCP surfaces + +- **`caido-mode`** — TypeScript CLI. Use for curl-through-Caido iteration, + Match & Replace rules, and replay-session/collection handoff to the operator. +- **`caido-sdk`** — Python library for in-process read/replay. + +### Credential redaction (`caido-go`) + +The Go server **redacts** `Authorization`, `Cookie`, `Set-Cookie`, and API-key +headers in all output by default. On an authorized engagement where you need +real values (to replay a captured authenticated request, or produce a working +`caido_export_curl` PoC), set `CAIDO_ALLOW_SENSITIVE_HEADERS=true` in the +server env. If a replayed request unexpectedly 401s, suspect redaction before +suspecting the target. + ## HTTPQL Quick Reference ``` @@ -34,49 +68,54 @@ resp.code.gte:500 ## MCP Tools -### Search history -`mcp__caido__list_requests` -- search proxy history with HTTPQL -- `httpql`: filter string -- `limit`: max results (default 20, max 100) - -### Get request details -`mcp__caido__get_request` -- full request/response -- `ids`: request ID array -- `include`: `["requestHeaders", "requestBody", "responseHeaders", "responseBody"]` - -### Replay -`mcp__caido__send_request` -- send raw HTTP request -- `raw`: full HTTP request text -- `host`: target host -- `port`: target port (default 443) -- `tls`: use HTTPS (default true) - -### Fuzzing results -`mcp__caido__list_automate_sessions` -- list fuzzing sessions -`mcp__caido__get_automate_session` -- session details -`mcp__caido__get_automate_entry` -- fuzzing results with pagination - -### Findings -`mcp__caido__create_finding` -- document a security finding -- `requestId`: associated request ID -- `title`: finding title -- `description`: detailed description +> **Check your live tool schema first.** Names below are the complete surface of +> the **`caido` (Python)** server, taken from `mcp/caido.py`. Client runtimes +> namespace them differently (bare `caido_health`, or prefixed +> `mcp__caido__health`) — match whatever your schema actually shows. Anything +> not in this table lives on **`caido-go`**. + +### `caido` (Python) — the full list, 9 tools + +| Tool | Arguments | +|---|---| +| `caido_health` | — | +| `caido_search_requests` | `filter` (HTTPQL), `limit` = 20 | +| `caido_get_request` | `request_id`, `include` (comma string: `headers,body`) | +| `caido_replay_request` | `raw_request`, `host`, `port` = auto, `tls` = true | +| `caido_list_scopes` | — | +| `caido_create_scope` | `name`, `allowlist[]`, `denylist[]` | +| `caido_list_findings` | `filter`, `limit` = 20 | +| `caido_create_finding` | `request_id`, `title`, `description`, `reporter` = `dreadnode-agent`, `dedupe_key` | +| `caido_replay_sessions` | `limit` = 20 | + +Note the argument style: `request_id` (singular, snake_case) and a comma-joined +`include` string — **not** an `ids` array or an `include` list. `caido_get_request` +fetches one request at a time. + +### On `caido-go` only + +Fuzzing/Automate (`caido_list_automate_sessions`, `caido_get_automate_session`, +`caido_get_automate_entry`), batch send, race windows, intercept, environments, +filter presets, tamper rules, sitemap, projects, workflows, WebSocket streams, +and `caido_export_curl`. If one of these is missing from your schema, the Go +server isn't connected — fall back to the equivalent Python tool or the +`caido-mode` skill. ## Common Workflows -### IDOR validation +### IDOR validation (`caido` Python server) ``` -1. Search: mcp__caido__list_requests(httpql: 'req.path.cont:"/api/" AND req.method.eq:"GET"', limit: 50) -2. Inspect: mcp__caido__get_request(ids: [""], include: ["requestHeaders","requestBody","responseHeaders","responseBody"]) -3. Replay with modified ID: mcp__caido__send_request(raw: "", host: "target.com") -4. Document: mcp__caido__create_finding(requestId: "", title: "IDOR in /api/users/{id}") +1. Search: caido_search_requests(filter: 'req.path.cont:"/api/" AND req.method.eq:"GET"', limit: 50) +2. Inspect: caido_get_request(request_id: "", include: "headers,body") +3. Replay: caido_replay_request(raw_request: "", host: "target.com") +4. Document: caido_create_finding(request_id: "", title: "IDOR in /api/users/{id}") ``` -### Fuzzing result triage +### Fuzzing result triage (requires `caido-go`) ``` -1. mcp__caido__list_automate_sessions() -2. mcp__caido__get_automate_session(id: "") -3. mcp__caido__get_automate_entry(id: "", limit: 20) +1. caido_list_automate_sessions() +2. caido_get_automate_session(id: "") +3. caido_get_automate_entry(id: "", limit: 20) Compare response sizes/codes for anomalies ``` @@ -84,12 +123,19 @@ resp.code.gte:500 | Error | Fix | |---|---| -| `Invalid token` | Run `caido-mcp-server login` | -| `Connection refused` | Start Caido desktop app | -| `No such tool` | Check capability MCP config | +| `Invalid token` | The Go server needs the **local instance access token**, not a Caido Cloud PAT (`caido_...`). Grab it in the Caido GUI devtools console: `JSON.parse(localStorage.CAIDO_AUTHENTICATION).accessToken` → `CAIDO_ACCESS_TOKEN`. Or run `caido-mcp-server login` for OAuth (auto-refreshes). | +| `token expired, no refresh token` | Static access tokens last ~7 days; re-grab it, or switch to `caido-mcp-server login`. | +| `Unknown field collection on type ReplaySession` | Python `caido-sdk-client` < 0.3.0 against Caido >= 0.57. Upgrade to `>= 0.3.0`. | +| `Connection refused` | Caido isn't running, or `CAIDO_URL` points at the wrong port. | +| `No such tool` | Tool lives on the other Caido MCP server — check the table above. | +| `poll failed: timed out` | Target slow; fetch the result with the returned entry id. | ```bash -lsof -i :8080 # Caido running? -which caido-mcp-server # Binary exists? -cat ~/.config/caido-mcp-server/token # Authenticated? +lsof -nP -i :8080 -sTCP:LISTEN # Caido listening? (read-only check) +which caido-mcp-server # Go MCP binary on PATH? +ls ~/.caido-mcp/token.json # Shared OAuth token present? ``` + +> Diagnose read-only. Never kill a Caido process to "reset" it — the desktop app +> runs its backend as `caido-cli --listen`, so a broad `pkill` takes down the +> operator's running instance and their in-flight project state. diff --git a/capabilities/web-security/skills/caido-sdk/SKILL.md b/capabilities/web-security/skills/caido-sdk/SKILL.md index b00399e..896f533 100644 --- a/capabilities/web-security/skills/caido-sdk/SKILL.md +++ b/capabilities/web-security/skills/caido-sdk/SKILL.md @@ -1,6 +1,7 @@ --- name: caido-sdk -description: "Direct Caido interaction via the caido-sdk-client Python library, bypassing the Caido MCP server. Prefer this over the caido-proxy MCP skill for efficiency WHEN the SDK is importable in the current runtime. If the import fails, or Caido/the MCP is not loaded, fall back to the caido-proxy skill." +description: "Direct Caido interaction via the caido-sdk-client Python library, bypassing the Caido MCP server. Prefer this over the caido-proxy MCP skill for efficiency WHEN the SDK is importable in the current runtime. If the import fails, or Caido/the MCP is not loaded, fall back to the caido-proxy skill. For curl-through-Caido testing, match & replace rules, or replay handoffs, use caido-mode instead." +compatibility: "Requires caido-sdk-client >= 0.3.0 (importable or via uv) and a reachable Caido instance (CAIDO_URL)." --- # Caido SDK (direct) @@ -9,6 +10,29 @@ Talk to a running Caido instance directly through `caido-sdk-client` — one process, no per-call MCP round-trips. Preferred over the `caido-proxy` MCP skill **only when the library is importable**. +> **Sibling surfaces.** This capability ships four Caido surfaces against one +> instance. Use **this** skill for in-process Python read/replay. For +> curl-through-Caido testing, Match & Replace rules, or replay handoff, use +> **`caido-mode`**. For tool-call access without writing code, use +> **`caido-proxy`** (`caido` MCP for the basics, `caido-go` MCP for batch send, +> race windows, intercept, environments, WS). They share `CAIDO_URL`; only +> `caido-mode` uses a separate credential store, so none of them clobber each +> other's auth. + +## Version floor (important) + +Requires **`caido-sdk-client >= 0.3.0`**. Earlier releases (0.2.x) hardcode the +pre-0.57 replay schema — they select `collection`/`activeEntry` on +`ReplaySession` and omit `ReplaySessionKind` — so replay calls fail against +Caido >= 0.57 with `Unknown field collection on type ReplaySession`. 0.3.0 added +the versioned transport split (`transport/latest` vs `transport/v0_56`) and +negotiates the right schema per instance. + +```bash +python3 -c "import importlib.metadata as m; print(m.version('caido-sdk-client'))" +# < 0.3.0 and talking to Caido >= 0.57? use: uv run --with 'caido-sdk-client>=0.3.0' script.py +``` + ## Step 0 — availability (probe first, never assume) The SDK usually lives only inside the MCP's isolated env, not the agent runtime. @@ -17,8 +41,8 @@ The SDK usually lives only inside the MCP's isolated env, not the agent runtime. python3 -c "import caido_sdk_client" 2>/dev/null && echo "USE SDK" || echo "NO SDK" ``` -1. Import works → use the SDK (below). -2. Import fails but `uv` is on PATH → `uv run --with caido-sdk-client script.py`. +1. Import works → check the version floor above, then use the SDK (below). +2. Import fails but `uv` is on PATH → `uv run --with 'caido-sdk-client>=0.3.0' script.py`. 3. Neither, or Caido unreachable → load the **`caido-proxy`** skill, use `caido_*` MCP tools. ## Auth (resolution order) @@ -62,12 +86,19 @@ async def main(): if entry and entry.response and entry.response.raw: print(entry.response.raw.decode(errors="replace")[:2000]) - # replay - from caido_sdk_client.types.replay_session import ReplaySendOptions - s = await client.replay.sessions.create() - res = await client.replay.send(s.id, ReplaySendOptions( - raw=b"GET / HTTP/1.1\r\nHost: example.com\r\n\r\n", host="example.com", port=443, tls=True)) - print(res.task_status) + # replay — SEED the session, and bound the send (see notes below) + from caido_sdk_client.types.replay_session import ( + CreateReplaySessionFromRaw, CreateReplaySessionOptions, ReplaySendOptions) + from caido_sdk_client.types.network import ConnectionInfoInput + raw_req = b"GET / HTTP/1.1\r\nHost: example.com\r\n\r\n" + conn = ConnectionInfoInput(host="example.com", port=443, is_tls=True) + s = await client.replay.sessions.create(CreateReplaySessionOptions( + request_source=CreateReplaySessionFromRaw(raw=raw_req, connection=conn))) + res = await asyncio.wait_for( + client.replay.send(s.id, ReplaySendOptions(raw=raw_req, connection=conn)), 30) + print(getattr(res.status, "value", res.status)) # "DONE" | "CANCELLED" | "ERROR" + if res.entry and res.entry.response: + print(res.entry.response.status_code) # finding from caido_sdk_client.types.finding import CreateFindingOptions @@ -78,6 +109,63 @@ async def main(): asyncio.run(main()) ``` +## API shapes that bite + +Three mistakes account for most runtime `TypeError`s here. The signatures below +are verified against `caido-sdk-client` 0.3.0: + +| Wrong | Right | +|---|---| +| `ReplaySendOptions(raw=…, host=…, port=…, tls=…)` | `ReplaySendOptions(raw=…, connection=ConnectionInfoInput(host=…, port=…, is_tls=…))` | +| `ConnectionInfoInput(…, tls=True)` | `ConnectionInfoInput(…, is_tls=True)` | +| `result.task_status` | `result.status` — `"DONE"` / `"CANCELLED"` / `"ERROR"` | + +`ReplaySendOptions` fields are exactly `raw`, `connection`, `settings`. +`ReplaySendResult` fields are exactly `entry`, `status`, `error`. + +### Two more, both verified live against Caido 0.57.1 + +**Seed the session, or `send()` refuses.** On >= 0.57, `send()` updates the +draft of an *existing* entry and then starts a replay task. A bare +`sessions.create()` produces a session with no entries, so `send()` raises +`OtherUserError: Replay session has no entries`. Create the session with +`request_source=CreateReplaySessionFromRaw(raw=..., connection=...)`. + +**Always bound `send()` with `asyncio.wait_for`.** After starting the task the +SDK waits on a task-finished *subscription*. If the target answers before that +subscription is established, the event is missed and the await never returns — +reproducible roughly 2 in 3 times against a localhost target. The request is +still sent, so on timeout recover the result instead of assuming failure: + +```python +try: + res = await asyncio.wait_for(client.replay.send(s.id, opts), 30) + entry_id = res.entry.id +except TimeoutError: + session = await client.replay.sessions.get(s.id) + conn = await session.entries().last(1).execute() + entry_id = conn.edges[-1].node.id + +# Re-fetch by id — entries listed off a session carry no response body, +# and `response` hangs off the ENTRY, not off entry.request. +entry = await client.replay.entries.get(entry_id) +print(entry.response.status_code, entry.response.raw[:200]) +``` + +`status` may arrive as a `TaskStatus` enum whose `str()` is `"TaskStatus.DONE"` +— unwrap with `getattr(status, "value", status)`. + +Confirm against whatever is actually installed before writing a long script: + +```bash +python3 -c " +import dataclasses as d +from caido_sdk_client.types.replay_session import ReplaySendOptions, ReplaySendResult +from caido_sdk_client.types.network import ConnectionInfoInput +for c in (ReplaySendOptions, ReplaySendResult, ConnectionInfoInput): + print(c.__name__, [f.name for f in d.fields(c)])" +``` + ## Notes - HTTPQL filter syntax is identical to the `caido-proxy` skill; see its reference. diff --git a/capabilities/web-security/tests/test_caido_mcp.py b/capabilities/web-security/tests/test_caido_mcp.py index 219055c..60c343a 100644 --- a/capabilities/web-security/tests/test_caido_mcp.py +++ b/capabilities/web-security/tests/test_caido_mcp.py @@ -44,16 +44,22 @@ def _stub_caido_sdk(monkeypatch: pytest.MonkeyPatch) -> None: replay_mod = types.ModuleType("caido_sdk_client.types.replay_session") replay_mod.ReplaySendOptions = MagicMock # type: ignore[attr-defined] + replay_mod.CreateReplaySessionFromRaw = MagicMock # type: ignore[attr-defined] + replay_mod.CreateReplaySessionOptions = MagicMock # type: ignore[attr-defined] scope_mod = types.ModuleType("caido_sdk_client.types.scope") scope_mod.CreateScopeOptions = MagicMock # type: ignore[attr-defined] + network_mod = types.ModuleType("caido_sdk_client.types.network") + network_mod.ConnectionInfoInput = MagicMock # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "caido_sdk_client", sdk) monkeypatch.setitem(sys.modules, "caido_sdk_client.auth", auth) monkeypatch.setitem(sys.modules, "caido_sdk_client.types", types.ModuleType("caido_sdk_client.types")) monkeypatch.setitem(sys.modules, "caido_sdk_client.types.finding", finding_mod) monkeypatch.setitem(sys.modules, "caido_sdk_client.types.replay_session", replay_mod) monkeypatch.setitem(sys.modules, "caido_sdk_client.types.scope", scope_mod) + monkeypatch.setitem(sys.modules, "caido_sdk_client.types.network", network_mod) def _load_caido_module() -> types.ModuleType: @@ -186,3 +192,126 @@ async def tracking_sleep(delay: float) -> None: await client.safe_get() assert sleep_calls == [2.0] + + +# ============================================================================= +# SDK API contract (static) +# ============================================================================= + + +class TestCaidoSdkApiContract: + """Guard against silent drift between mcp/caido.py and caido-sdk-client. + + The fixture above stubs the SDK dataclasses with ``MagicMock``, which + accepts *any* keyword argument. That is fine for exercising retry logic, + but it means a genuinely wrong constructor call (``ReplaySendOptions( + host=..., port=..., tls=...)`` instead of the nested ``connection= + ConnectionInfoInput(...)``) sails through the mocked tests and only fails + against a real instance. + + These tests read the source with ``ast`` and assert the shape directly, so + they hold regardless of what is installed in the test environment. + """ + + SOURCE = MODULE_PATH.read_text(encoding="utf-8") + + def _calls(self, name: str) -> list[set[str]]: + import ast + + found: list[set[str]] = [] + for node in ast.walk(ast.parse(self.SOURCE)): + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == name + ): + found.append({kw.arg for kw in node.keywords if kw.arg}) + return found + + def test_replay_send_options_uses_nested_connection(self) -> None: + calls = self._calls("ReplaySendOptions") + assert calls, "ReplaySendOptions is never constructed" + for kwargs in calls: + # Real fields are raw / connection / settings. + assert kwargs <= {"raw", "connection", "settings"}, ( + f"unexpected kwargs {sorted(kwargs)}; connection details belong " + "in ConnectionInfoInput, not flat on ReplaySendOptions" + ) + assert "connection" in kwargs + + def test_connection_info_input_field_names(self) -> None: + calls = self._calls("ConnectionInfoInput") + assert calls, "ConnectionInfoInput is never constructed" + for kwargs in calls: + assert kwargs <= {"host", "port", "is_tls", "sni"} + # `is_tls`, not `tls` — a silent TypeError at runtime otherwise. + assert "tls" not in kwargs + + def test_connection_info_input_is_imported(self) -> None: + assert ( + "from caido_sdk_client.types.network import ConnectionInfoInput" + in self.SOURCE + ) + + def test_replay_result_status_attribute(self) -> None: + # ReplaySendResult exposes `status`, never `task_status`. + assert "task_status" not in self.SOURCE + + def test_pep723_header_pins_sdk_floor(self) -> None: + # The uv-run script header must carry the same floor as capability.yaml. + assert '"caido-sdk-client>=0.3.0"' in self.SOURCE + + # -- live-verified against Caido 0.57.1 ------------------------------- + # + # These assertions inspect the body of `caido_replay_request` via AST + # rather than scanning the whole file, so prose and comments that mention + # `replay.send()` cannot satisfy (or break) them. + + @staticmethod + def _tool_body(name: str) -> str: + import ast + + tree = ast.parse(TestCaidoSdkApiContract.SOURCE) + for node in tree.body: + if isinstance(node, ast.AsyncFunctionDef) and node.name == name: + return ast.unparse(node) + raise AssertionError(f"{name} not found") + + def test_replay_session_is_seeded_with_the_request(self) -> None: + # On Caido >= 0.57, replay.send() updates the draft of an EXISTING + # entry. A bare sessions.create() yields an empty session and send() + # aborts with "Replay session has no entries" - verified live against + # 0.57.1. The session must be created from the raw request. + body = self._tool_body("caido_replay_request") + assert "CreateReplaySessionFromRaw" in body + assert "request_source=" in body + assert body.index("replay.sessions.create(") < body.index("replay.send(") + + def test_replay_send_is_bounded_by_a_timeout(self) -> None: + # send() waits on a task-finished subscription that is missed when the + # target responds faster than the subscription is established (~2 in 3 + # against localhost). Unbounded, the MCP call hangs forever. + body = self._tool_body("caido_replay_request") + assert "REPLAY_SEND_TIMEOUT" in self.SOURCE + assert "asyncio.wait_for(" in body + assert "timeout=REPLAY_SEND_TIMEOUT" in body + + def test_replay_timeout_falls_back_to_session_state(self) -> None: + # The request really was sent, so a timeout must recover the result + # rather than report a false failure. + body = self._tool_body("caido_replay_request") + assert "TimeoutError" in body + assert "_replay_result_from_session" in body + + def test_recovery_reads_response_off_the_entry(self) -> None: + # Entries listed from a session carry no response body; the entry must + # be re-fetched by id, and `response` hangs off the entry itself, not + # off entry.request. + body = self._tool_body("_replay_result_from_session") + assert "replay.entries.get(" in body + assert "getattr(entry, 'response', None)" in body or 'getattr(entry, "response", None)' in body + + def test_status_enum_is_unwrapped(self) -> None: + # TaskStatus.DONE stringifies as "TaskStatus.DONE"; callers want "DONE". + body = self._tool_body("caido_replay_request") + assert "getattr(status_value, 'value', status_value)" in body or 'getattr(status_value, "value", status_value)' in body diff --git a/capabilities/web-security/tests/test_caido_mode_skill.py b/capabilities/web-security/tests/test_caido_mode_skill.py new file mode 100644 index 0000000..e8b5169 --- /dev/null +++ b/capabilities/web-security/tests/test_caido_mode_skill.py @@ -0,0 +1,419 @@ +"""Tests for the vendored `caido-mode` skill (Caido TypeScript SDK CLI). + +The web-security capability ships four Caido surfaces against one instance: + + * ``caido-cli`` — the headless Caido server binary. + * ``caido`` MCP — lightweight Python wrapper over ``caido-sdk-client``. + * ``caido-go`` MCP — full-surface upstream Go binary (see + ``test_caido_go_mcp.py``). + * ``caido-mode`` — this skill: a vendored TypeScript CLI built on + ``@caido/sdk-client`` (``caido-ts``). + +These tests lock the pieces that silently rot: the skill's SDK floor, the +matching server pin, the provision-time ``npm install``, the presence check, +and the Dreadnode skill-format frontmatter. They are pure static assertions +over the manifest, the install script, and the skill tree — no network, no +Node, and no running Caido instance required. + +Version contract +---------------- +``@caido/sdk-client`` 0.4.0 targets the Caido **0.57** replay schema +(``ReplaySession`` as an interface, ``kind: ReplaySessionKind!`` on +``createReplaySession``, task-based sending via ``startReplayTask``). Both the +server pin and the Python SDK floor must stay on that side of the break, or +replay silently fails with "Unknown field collection on type ReplaySession". +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import pytest +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +MANIFEST = yaml.safe_load((ROOT / "capability.yaml").read_text(encoding="utf-8")) +INSTALL_SCRIPT = (ROOT / "scripts" / "install_tools.sh").read_text(encoding="utf-8") +DOCKERFILE = (ROOT / "docker" / "Dockerfile.runtime").read_text(encoding="utf-8") + +SKILL_DIR = ROOT / "skills" / "caido-mode" +SKILL_MD = SKILL_DIR / "SKILL.md" +PACKAGE_JSON = json.loads((SKILL_DIR / "package.json").read_text(encoding="utf-8")) + +# Keep in lock-step with scripts/install_tools.sh. Bumping the server pin means +# bumping this constant, which forces a conscious re-check of the SDK contract. +CAIDO_SERVER_PIN = "0.57.1" + +# Minimum Caido server that speaks the replay schema @caido/sdk-client 0.4.0 +# expects. See the module docstring. +MIN_REPLAY_SCHEMA_SERVER = (0, 57) + +# Python caido-sdk-client floor: 0.3.0 introduced the versioned transport split +# (transport/latest vs transport/v0_56) that negotiates 0.57 correctly. +PY_SDK_FLOOR = "caido-sdk-client>=0.3.0" + + +def _frontmatter(path: Path) -> dict: + text = path.read_text(encoding="utf-8") + match = re.match(r"^---\s*\n(.*?)\n---\s*\n", text, re.DOTALL) + assert match is not None, f"missing YAML frontmatter in {path}" + data = yaml.safe_load(match.group(1)) + assert isinstance(data, dict), f"frontmatter must be a mapping in {path}" + return data + + +def _version_tuple(spec: str) -> tuple[int, ...]: + return tuple(int(p) for p in re.findall(r"\d+", spec)[:3]) + + +# ============================================================================= +# Skill tree +# ============================================================================= + + +class TestCaidoModeSkillTree: + def test_skill_md_exists(self) -> None: + assert SKILL_MD.is_file() + + def test_cli_entrypoint_is_vendored(self) -> None: + assert (SKILL_DIR / "caido-client.ts").is_file() + + def test_command_modules_are_vendored(self) -> None: + # The command surface the SKILL.md documents must actually be present. + commands = SKILL_DIR / "lib" / "commands" + for module in ("requests", "replay", "matchreplace", "findings", "info"): + assert (commands / f"{module}.ts").is_file(), f"missing {module}.ts" + + def test_upstream_tests_are_vendored(self) -> None: + tests = sorted(p.name for p in (SKILL_DIR / "test").glob("*.test.ts")) + assert tests == [ + "exportcurl.test.ts", + "matchreplace.test.ts", + "rawedit.test.ts", + ] + + def test_node_modules_not_committed(self) -> None: + # 25 MB of node_modules must never enter the artifact; it is recreated + # at provision time. Both .gitignore and the OCI packager exclude it. + gitignore = (SKILL_DIR / ".gitignore").read_text(encoding="utf-8") + assert "node_modules" in gitignore + + +# ============================================================================= +# Dreadnode skill format +# ============================================================================= + + +class TestCaidoModeFrontmatter: + def test_name_matches_directory(self) -> None: + assert _frontmatter(SKILL_MD)["name"] == "caido-mode" + + def test_description_within_loader_limit(self) -> None: + # dreadnode/agents/skills.py enforces SKILL_DESCRIPTION_MAX_LENGTH=1024. + description = _frontmatter(SKILL_MD)["description"] + assert 0 < len(description) <= 1024 + + def test_description_routes_against_sibling_surfaces(self) -> None: + # The router picks between four Caido surfaces on description alone, so + # this one must name its alternatives. + description = _frontmatter(SKILL_MD)["description"].lower() + assert "caido-sdk" in description + assert "caido-proxy" in description + + def test_no_unsupported_tags_key(self) -> None: + # Upstream ships `tags: [worker]`, which the Dreadnode loader silently + # drops. The equivalent lives in `metadata.role`. + frontmatter = _frontmatter(SKILL_MD) + assert "tags" not in frontmatter + assert frontmatter["metadata"]["role"] == "worker" + + def test_metadata_values_are_strings(self) -> None: + # The loader rejects non-string metadata keys/values outright. + for key, value in _frontmatter(SKILL_MD)["metadata"].items(): + assert isinstance(key, str) and isinstance(value, str) + + def test_declares_compatibility(self) -> None: + compatibility = _frontmatter(SKILL_MD)["compatibility"] + assert isinstance(compatibility, str) and compatibility.strip() + assert len(compatibility) <= 500 # SKILL_COMPATIBILITY_MAX_LENGTH + assert "Node" in compatibility + + def test_records_upstream_provenance(self) -> None: + # A vendored skill must say where it came from, or the next re-sync + # silently clobbers the local fork. + metadata = _frontmatter(SKILL_MD)["metadata"] + assert "caido/skills" in metadata["upstream"] + + def test_documents_local_fork_for_resync(self) -> None: + body = SKILL_MD.read_text(encoding="utf-8") + assert "VENDORED SKILL" in body + assert "Preserve these local sections on re-sync" in body + + +# ============================================================================= +# Version contract — the 0.57 replay schema break +# ============================================================================= + + +class TestCaidoVersionContract: + def test_skill_pins_sdk_client_major_minor(self) -> None: + spec = PACKAGE_JSON["dependencies"]["@caido/sdk-client"] + assert _version_tuple(spec)[:2] == (0, 4), ( + f"expected @caido/sdk-client 0.4.x, got {spec!r}. A major/minor bump " + "may move the replay schema — re-verify the server pin." + ) + + def test_server_pin_speaks_the_same_replay_schema(self) -> None: + match = re.search(r'CAIDO_VERSION="([\d.]+)"', INSTALL_SCRIPT) + assert match is not None, "caido-cli version pin not found" + assert match.group(1) == CAIDO_SERVER_PIN + assert _version_tuple(match.group(1))[:2] >= MIN_REPLAY_SCHEMA_SERVER, ( + "caido-cli pin predates the 0.57 replay schema that " + "@caido/sdk-client 0.4.0 targets" + ) + + def test_server_download_url_is_version_interpolated(self) -> None: + # Guards against a bumped constant with a hardcoded URL left behind. + assert ( + "https://caido.download/releases/v${CAIDO_VERSION}/" + "caido-cli-v${CAIDO_VERSION}-linux-${CAIDO_ARCH}.tar.gz" + ) in INSTALL_SCRIPT + + @pytest.mark.parametrize( + "source", + [ + pytest.param("manifest", id="capability.yaml"), + pytest.param("mcp", id="mcp/caido.py"), + pytest.param("dockerfile", id="Dockerfile.runtime"), + ], + ) + def test_python_sdk_floor_declared_everywhere(self, source: str) -> None: + # All three declaration sites must carry the floor; a bare + # "caido-sdk-client" resolves 0.2.0, which breaks replay on 0.57. + if source == "manifest": + haystack = "\n".join(MANIFEST["dependencies"]["python"]) + elif source == "mcp": + haystack = (ROOT / "mcp" / "caido.py").read_text(encoding="utf-8") + else: + haystack = DOCKERFILE + assert PY_SDK_FLOOR in haystack + + def test_no_unpinned_python_sdk_reference_remains(self) -> None: + # Catch a stray bare dependency line reintroducing the 0.2.0 resolution. + for spec in MANIFEST["dependencies"]["python"]: + if spec.startswith("caido-sdk-client"): + assert spec == PY_SDK_FLOOR + + +# ============================================================================= +# Provisioning contract +# ============================================================================= + + +class TestCaidoModeInstall: + def test_installs_node_deps_at_provision_time(self) -> None: + assert "npm install --no-audit --no-fund" in INSTALL_SCRIPT + + def test_resolves_skill_dir_from_capability_root(self) -> None: + # CAPABILITY_ROOT when exported, else the script's own parent — the + # skill dir is mounted, not baked into the image. + assert ( + 'CAIDO_MODE_DIR="${CAPABILITY_ROOT:-$(cd "$(dirname "$0")/.." && pwd)}' + "/skills/caido-mode\"" in INSTALL_SCRIPT + ) + + def test_install_is_guarded_on_package_json(self) -> None: + assert 'if [ -f "$CAIDO_MODE_DIR/package.json" ]; then' in INSTALL_SCRIPT + + def test_install_failure_is_non_fatal(self) -> None: + # A missing Node toolchain must not abort the whole provision run. + assert "WARN: caido-mode npm install failed, skipping" in INSTALL_SCRIPT + + def test_node_is_available_before_skill_install(self) -> None: + # npm must exist by the time the skill block runs. + node_setup = INSTALL_SCRIPT.index("deb.nodesource.com") + skill_install = INSTALL_SCRIPT.index("CAIDO_MODE_DIR=") + assert node_setup < skill_install + + def test_presence_check_registered(self) -> None: + checks = {c["name"]: c["command"] for c in MANIFEST["checks"]} + assert "caido-mode" in checks + command = checks["caido-mode"] + # Both the entrypoint and the installed deps — either alone is a + # false green. + assert "skills/caido-mode/caido-client.ts" in command + assert "skills/caido-mode/node_modules" in command + + +# ============================================================================= +# Cross-surface coexistence +# ============================================================================= + + +class TestCaidoSurfacesCoexist: + def test_all_four_surfaces_are_declared(self) -> None: + servers = MANIFEST["mcp"]["servers"] + checks = {c["name"] for c in MANIFEST["checks"]} + assert "caido" in servers and "caido-go" in servers + assert {"caido-cli", "caido-mcp-server", "caido-mode"} <= checks + + def test_sibling_caido_skills_present(self) -> None: + for name in ("caido-sdk", "caido-proxy"): + assert (ROOT / "skills" / name / "SKILL.md").is_file() + + def test_auth_stores_are_disjoint(self) -> None: + # caido-mode keeps its own credential store so `setup` here never + # clobbers the token file the Python SDK and both MCP servers share. + body = SKILL_MD.read_text(encoding="utf-8") + assert "~/.claude/config/secrets.json" in body + assert "~/.caido-mcp/token.json" in body + + def test_skill_documents_surface_routing(self) -> None: + body = SKILL_MD.read_text(encoding="utf-8") + assert "Which Caido surface should I use?" in body + for surface in ("caido-mode", "caido-sdk", "caido` MCP", "caido-go"): + assert surface in body + + +# ============================================================================= +# Sibling skill: caido-sdk example code +# ============================================================================= + + +class TestCaidoSdkSkillExample: + """The `caido-sdk` skill ships runnable Python that agents copy verbatim. + + A wrong signature there is worse than no example: it produces a confident + `TypeError` at runtime. These assertions mirror the ones guarding + ``mcp/caido.py`` in ``test_caido_mcp.py``. + + Verified against caido-sdk-client 0.3.0: + ReplaySendOptions(raw, connection, settings) + ConnectionInfoInput(host, port, is_tls, sni) + ReplaySendResult(entry, status, error) + """ + + SKILL = (ROOT / "skills" / "caido-sdk" / "SKILL.md").read_text(encoding="utf-8") + + def _python_blocks(self) -> list[str]: + return re.findall(r"```python\n(.*?)```", self.SKILL, re.DOTALL) + + def test_has_a_python_example(self) -> None: + assert self._python_blocks() + + def test_examples_are_syntactically_valid(self) -> None: + import ast + + for block in self._python_blocks(): + ast.parse(block) + + def test_example_sdk_constructor_kwargs_are_real(self) -> None: + import ast + + valid = { + "ReplaySendOptions": {"raw", "connection", "settings"}, + "ConnectionInfoInput": {"host", "port", "is_tls", "sni"}, + "CreateFindingOptions": {"title", "reporter", "description", "dedupe_key"}, + "CreateScopeOptions": {"name", "allowlist", "denylist"}, + } + seen = 0 + for block in self._python_blocks(): + for node in ast.walk(ast.parse(block)): + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id in valid + ): + seen += 1 + kwargs = {kw.arg for kw in node.keywords if kw.arg} + extra = kwargs - valid[node.func.id] + assert not extra, ( + f"{node.func.id}(...) in caido-sdk SKILL.md passes " + f"{sorted(extra)}, which the dataclass does not accept" + ) + assert seen, "no SDK constructions found to validate" + + def test_example_does_not_use_task_status(self) -> None: + # ReplaySendResult exposes .status, never .task_status. Checked against + # runnable code only — prose may name the wrong form to warn about it. + for block in self._python_blocks(): + assert "task_status" not in block + + def test_example_does_not_flatten_connection_kwargs(self) -> None: + for block in self._python_blocks(): + collapsed = " ".join(block.split()) + assert "ReplaySendOptions(" not in collapsed or "connection=" in collapsed + + def test_documents_the_python_sdk_floor(self) -> None: + assert "caido-sdk-client>=0.3.0" in self.SKILL or ">= 0.3.0" in self.SKILL + + +# ============================================================================= +# Sibling skill: caido-proxy tool names +# ============================================================================= + + +class TestCaidoProxySkillToolNames: + """The `caido-proxy` skill must describe tools that actually exist. + + It previously documented Go-server tool names (`list_requests`, + `send_request`, the Automate family) as if they were on the Python server, + under an `mcp__caido__` prefix that matched neither. An agent following it + calls a tool that isn't in its schema. + """ + + SKILL = (ROOT / "skills" / "caido-proxy" / "SKILL.md").read_text(encoding="utf-8") + + @staticmethod + def _python_server_tools() -> set[str]: + import ast + + source = (ROOT / "mcp" / "caido.py").read_text(encoding="utf-8") + return { + node.name + for node in ast.parse(source).body + if isinstance(node, ast.AsyncFunctionDef) + and any(getattr(d, "attr", "") == "tool" for d in node.decorator_list) + } + + def test_every_python_server_tool_is_documented(self) -> None: + missing = {t for t in self._python_server_tools() if t not in self.SKILL} + assert not missing, f"undocumented `caido` tools: {sorted(missing)}" + + def test_no_stale_mcp_double_underscore_names(self) -> None: + # `mcp__caido__list_requests` matched no real tool on either server. + # The prefix may still be *named* when explaining namespacing, but it + # must never be attached to a concrete tool name. + stale = { + match + for match in re.findall(r"mcp__caido__(\w+)", self.SKILL) + if match != "health" # sole allowed mention: the namespacing example + } + assert not stale, f"stale mcp__caido__ tool references: {sorted(stale)}" + + def test_go_only_tools_are_marked_as_such(self) -> None: + # Automate lives on caido-go; the skill must not imply the Python + # server provides it. + go_section = self.SKILL.index("On `caido-go` only") + for tool in ( + "caido_list_automate_sessions", + "caido_get_automate_session", + "caido_get_automate_entry", + ): + assert tool in self.SKILL + assert self.SKILL.index(tool) > go_section, ( + f"{tool} is a caido-go tool but appears before the caido-go section" + ) + + def test_documents_both_servers(self) -> None: + assert "Two MCP servers, one instance" in self.SKILL + assert "caido-go" in self.SKILL + + def test_warns_against_killing_caido_processes(self) -> None: + # Troubleshooting must not suggest pkill: the desktop app runs its + # backend as `caido-cli --listen`, so a broad kill takes down the + # operator's live instance and in-flight project state. + assert "pkill" in self.SKILL and "Never kill a Caido process" in self.SKILL diff --git a/capabilities/web-security/tests/test_caido_tools.py b/capabilities/web-security/tests/test_caido_tools.py deleted file mode 100644 index f8439d3..0000000 --- a/capabilities/web-security/tests/test_caido_tools.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Tests for the temporary Caido compatibility stub.""" - -from __future__ import annotations - -import importlib.util -from pathlib import Path - - -MODULE_PATH = Path(__file__).resolve().parent.parent / "tools" / "caido_proxy.py" -SPEC = importlib.util.spec_from_file_location("caido_proxy", MODULE_PATH) -assert SPEC and SPEC.loader -MODULE = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(MODULE) - - -def test_caido_proxy_stub_exports_no_toolset() -> None: - assert getattr(MODULE, "CaidoTools", None) is None diff --git a/capabilities/web-security/tools/caido_proxy.py b/capabilities/web-security/tools/caido_proxy.py deleted file mode 100644 index 569024f..0000000 --- a/capabilities/web-security/tools/caido_proxy.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Compatibility stub for the removed Caido Toolset integration. - -Caido is exposed through `mcp/caido.py`; this module exists so older imports do -not fail while making it explicit that no legacy Toolset is exported. -""" - -CaidoTools = None