From da24902c79936aa459bf3623ec164316d83bdee8 Mon Sep 17 00:00:00 2001 From: naamaz Date: Tue, 11 Aug 2026 13:57:52 +0300 Subject: [PATCH 1/6] Add gen_spec_v2: a spec generator with system vars and pre/post-tool triggers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gen_spec_v2 is an alternative to gen_spec that answers questions v1's schema cannot express: which system variables a rule depends on, whether it must be checked before or after the tool runs, what tool lookups or chat history it needs, and what the user has to supply before it can be enforced at all. Output format follows smith's spec_generation byte-for-byte (fixed key order, omit-when-empty), so specs are interchangeable with that corpus. New per-item fields: trigger pre_tool | post_tool requires system_vars / tool_history / message_history pending_for_user missing_tool | missing_var | clarification resolved_by_user the answers, once given conflicts contradictions across the spec set debug tool_info, archive (rejected items + why), notes Four entry points, because the caller knows when a complete set exists: generate_guard_specs_v2 one spec per tool, nothing else generate_spec_conflicts_v2 conflicts across a complete set generate_guard_specs_v2_full both generate_guard_examples_v2 re-run only the examples stage System variables come from a dict or a sys_var.json path. Values of any shape are kept (nested mappings and lists included); the agent's own action catalog (action_list, action_description) is excluded as it describes tools, not the acting user. The rendered block gives the model the exact access path (input.extensions.subject.) and domain, and code drops any variable the model invents but the file does not declare. Codegen path: specs_v2_to_v1 adapts v2 to the v1 schema gen_py already consumes, deriving skip = pending_for_user OR message_history OR post_tool OR system_vars. gen_py is unchanged. Reference grounding (refmatch) snaps each quoted reference back to the exact policy-document substring via a normalized-to-original offset map, with a difflib fallback. An item whose references ground in nothing is archived rather than kept, so a rule quoted from a tool docstring cannot masquerade as policy. Tests: 273 pass and 13 skip on a fresh checkout (the 13 need the employee ground-truth corpus under tests/data/, which is not committed). e2e coverage mirrors v1's calculator variants (functions, methods, langchain, OpenAPI dict), plus tau2 airline and an opt-in v1-vs-v2 guard-set delta report (-m delta). Known issue: on tau2 airline, a one-sentence policy can still bind to tools it does not govern (over-attachment). review_votes=5 is the current mitigation; see docs/superpowers/evidence/2026-08-11-gen-spec-v2-test-baseline.md. Backward compatibility: v1's loader accepts v2 files while ignoring the v2 fields, which yields all skip=False — point v2 at its own work_dir. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 100 ++++ .../2026-08-11-gen-spec-v2-test-baseline.md | 197 +++++++ .../specs/2026-08-10-gen-spec-v2-design.md | 482 ++++++++++++++++++ pyproject.toml | 5 + src/toolguard/buildtime/__init__.py | 21 + src/toolguard/buildtime/buildtime.py | 9 + .../buildtime/gen_spec_v2/__init__.py | 85 +++ .../buildtime/gen_spec_v2/adapter.py | 147 ++++++ .../buildtime/gen_spec_v2/conflicts.py | 118 +++++ .../buildtime/gen_spec_v2/context.py | 98 ++++ src/toolguard/buildtime/gen_spec_v2/models.py | 201 ++++++++ .../buildtime/gen_spec_v2/pipeline.py | 262 ++++++++++ .../buildtime/gen_spec_v2/prompts/__init__.py | 159 ++++++ .../gen_spec_v2/prompts/conflicts.txt | 34 ++ .../buildtime/gen_spec_v2/prompts/create.txt | 68 +++ .../buildtime/gen_spec_v2/prompts/enrich.txt | 56 ++ .../gen_spec_v2/prompts/examples.txt | 22 + .../buildtime/gen_spec_v2/prompts/expand.txt | 43 ++ .../buildtime/gen_spec_v2/prompts/review.txt | 51 ++ .../buildtime/gen_spec_v2/reconcile.py | 153 ++++++ .../buildtime/gen_spec_v2/refmatch.py | 302 +++++++++++ .../buildtime/gen_spec_v2/serialize.py | 159 ++++++ .../buildtime/gen_spec_v2/stages/__init__.py | 9 + .../buildtime/gen_spec_v2/stages/_shared.py | 87 ++++ .../buildtime/gen_spec_v2/stages/create.py | 50 ++ .../buildtime/gen_spec_v2/stages/enrich.py | 111 ++++ .../buildtime/gen_spec_v2/stages/examples.py | 56 ++ .../buildtime/gen_spec_v2/stages/expand.py | 67 +++ .../buildtime/gen_spec_v2/stages/review.py | 84 +++ .../buildtime/gen_spec_v2/sysvars.py | 118 +++++ .../buildtime/gen_spec_v2/tools_input.py | 45 ++ tests/buildtime/e2e/__init__.py | 0 .../buildtime/e2e/test_gen_spec_v2_codegen.py | 197 +++++++ tests/buildtime/e2e/test_guard_set_delta.py | 175 +++++++ tests/buildtime/e2e/test_tau2_v2.py | 264 ++++++++++ tests/buildtime/gen_spec_v2/__init__.py | 0 tests/buildtime/gen_spec_v2/conftest.py | 113 ++++ tests/buildtime/gen_spec_v2/test_adapter.py | 186 +++++++ tests/buildtime/gen_spec_v2/test_conflicts.py | 177 +++++++ tests/buildtime/gen_spec_v2/test_context.py | 66 +++ .../gen_spec_v2/test_examples_only.py | 165 ++++++ .../gen_spec_v2/test_gen_py_contract.py | 156 ++++++ tests/buildtime/gen_spec_v2/test_pipeline.py | 310 +++++++++++ tests/buildtime/gen_spec_v2/test_prompts.py | 112 ++++ tests/buildtime/gen_spec_v2/test_reconcile.py | 201 ++++++++ tests/buildtime/gen_spec_v2/test_refmatch.py | 249 +++++++++ .../gen_spec_v2/test_refmatch_real_policy.py | 48 ++ tests/buildtime/gen_spec_v2/test_serialize.py | 50 ++ tests/buildtime/gen_spec_v2/test_stages.py | 432 ++++++++++++++++ tests/buildtime/gen_spec_v2/test_sysvars.py | 127 +++++ .../buildtime/gen_spec_v2/test_tools_input.py | 64 +++ 51 files changed, 6491 insertions(+) create mode 100644 docs/superpowers/evidence/2026-08-11-gen-spec-v2-test-baseline.md create mode 100644 docs/superpowers/specs/2026-08-10-gen-spec-v2-design.md create mode 100644 src/toolguard/buildtime/gen_spec_v2/__init__.py create mode 100644 src/toolguard/buildtime/gen_spec_v2/adapter.py create mode 100644 src/toolguard/buildtime/gen_spec_v2/conflicts.py create mode 100644 src/toolguard/buildtime/gen_spec_v2/context.py create mode 100644 src/toolguard/buildtime/gen_spec_v2/models.py create mode 100644 src/toolguard/buildtime/gen_spec_v2/pipeline.py create mode 100644 src/toolguard/buildtime/gen_spec_v2/prompts/__init__.py create mode 100644 src/toolguard/buildtime/gen_spec_v2/prompts/conflicts.txt create mode 100644 src/toolguard/buildtime/gen_spec_v2/prompts/create.txt create mode 100644 src/toolguard/buildtime/gen_spec_v2/prompts/enrich.txt create mode 100644 src/toolguard/buildtime/gen_spec_v2/prompts/examples.txt create mode 100644 src/toolguard/buildtime/gen_spec_v2/prompts/expand.txt create mode 100644 src/toolguard/buildtime/gen_spec_v2/prompts/review.txt create mode 100644 src/toolguard/buildtime/gen_spec_v2/reconcile.py create mode 100644 src/toolguard/buildtime/gen_spec_v2/refmatch.py create mode 100644 src/toolguard/buildtime/gen_spec_v2/serialize.py create mode 100644 src/toolguard/buildtime/gen_spec_v2/stages/__init__.py create mode 100644 src/toolguard/buildtime/gen_spec_v2/stages/_shared.py create mode 100644 src/toolguard/buildtime/gen_spec_v2/stages/create.py create mode 100644 src/toolguard/buildtime/gen_spec_v2/stages/enrich.py create mode 100644 src/toolguard/buildtime/gen_spec_v2/stages/examples.py create mode 100644 src/toolguard/buildtime/gen_spec_v2/stages/expand.py create mode 100644 src/toolguard/buildtime/gen_spec_v2/stages/review.py create mode 100644 src/toolguard/buildtime/gen_spec_v2/sysvars.py create mode 100644 src/toolguard/buildtime/gen_spec_v2/tools_input.py create mode 100644 tests/buildtime/e2e/__init__.py create mode 100644 tests/buildtime/e2e/test_gen_spec_v2_codegen.py create mode 100644 tests/buildtime/e2e/test_guard_set_delta.py create mode 100644 tests/buildtime/e2e/test_tau2_v2.py create mode 100644 tests/buildtime/gen_spec_v2/__init__.py create mode 100644 tests/buildtime/gen_spec_v2/conftest.py create mode 100644 tests/buildtime/gen_spec_v2/test_adapter.py create mode 100644 tests/buildtime/gen_spec_v2/test_conflicts.py create mode 100644 tests/buildtime/gen_spec_v2/test_context.py create mode 100644 tests/buildtime/gen_spec_v2/test_examples_only.py create mode 100644 tests/buildtime/gen_spec_v2/test_gen_py_contract.py create mode 100644 tests/buildtime/gen_spec_v2/test_pipeline.py create mode 100644 tests/buildtime/gen_spec_v2/test_prompts.py create mode 100644 tests/buildtime/gen_spec_v2/test_reconcile.py create mode 100644 tests/buildtime/gen_spec_v2/test_refmatch.py create mode 100644 tests/buildtime/gen_spec_v2/test_refmatch_real_policy.py create mode 100644 tests/buildtime/gen_spec_v2/test_serialize.py create mode 100644 tests/buildtime/gen_spec_v2/test_stages.py create mode 100644 tests/buildtime/gen_spec_v2/test_sysvars.py create mode 100644 tests/buildtime/gen_spec_v2/test_tools_input.py diff --git a/README.md b/README.md index b801d0f..639c342 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,98 @@ The following generation phases can be selected via `spec_steps`: - `REVIEW_POLICIES_SELF_CONTAINED` – Ensure each policy description is fully self-contained and unambiguous. - `REVIEW_POLICIES_FEASIBILITY` – Validate that each policy can be deterministically enforced. +##### Alternative: v2 Specification Generation + +`gen_spec_v2` is a second, parallel generator. Where the generator above records a +rule's text and examples, v2 also records **who** is acting, **when** the rule applies, +**what else** is needed to decide it, and **what is missing** to enforce it at all — so +rules that today's generator silently drops become visible instead. + +It is an alternative, not a replacement: `generate_guard_specs` is unchanged and remains +the default. + +```python +from toolguard.buildtime import ( + generate_guard_specs_v2_full, + specs_v2_to_v1, + generate_guards_code, +) + +specs = await generate_guard_specs_v2_full( + policy_text=policy_text, # raw markdown; no bullet structure required + tools=tools, # functions, an OpenAPI dict, or a list[ToolInfo] + llm=llm, + work_dir="specs_v2", # give v2 its own directory (see the note below) + system_vars="sys_var.json", # a dict or a path; optional + source_doc="policy_doc.md", +) + +# Feed the existing code generator: +v1_specs = specs_v2_to_v1(specs, known_tools=[t.__name__ for t in tools]) +guards = await generate_guards_code( + tool_specs=v1_specs, tools=tools, work_dir="code", llm=llm, app_name="myapp" +) +``` + +###### System variables + +`system_vars` declares the attributes of the acting user that policies may refer to, +which is what makes rules like "only HR may add an employee" expressible. Pass a dict or +a path to a JSON file: + +```json +{ + "user_name": "Bob", + "user_id": 1, + "department": ["Corporate Leadership", "Engineering", "Product", "HR", "Finance"], + "organization": ["IBM Corporation", "Red Hat", "Kyndryl"] +} +``` + +A list value is a closed set of allowed values; anything else is one example of the shape +to expect. Nested values are fine — a structured attribute of the acting user is still a +subject variable. Two keys are ignored if present, `action_list` and +`action_description`, because they describe the agent's own tools rather than the acting +user. Generated specs may only name variables declared here. + +###### What each policy item records + +| Field | Meaning | +|---|---| +| `trigger` | `pre_tool` (decided from the arguments) or `post_tool` (decided against the result) | +| `requires.system_vars` | Which acting-user attributes the rule reads | +| `requires.tool_history` | A tool that must be called first, and with which arguments | +| `requires.message_history` | The rule can only be decided from the conversation | +| `pending_for_user` | A `missing_tool`, `missing_var`, or `clarification` gap, with the question to ask | +| `references` | Verbatim spans of the policy document the rule came from | + +###### Entry points + +| Function | Does | +|---|---| +| `generate_guard_specs_v2` | One spec per tool → `.json`. No cross-tool work. | +| `generate_spec_conflicts_v2` | Finds conflicts across a complete spec set (loaded from `work_dir` if not passed) and records them on each spec. Idempotent. | +| `generate_guard_specs_v2_full` | All tools, then conflicts — one call for a full build. | +| `generate_guard_examples_v2` | Reruns only the examples stage for specs already on disk, leaving everything else on each item untouched (the v2 counterpart of `generate_guard_examples`). | + +`SpecV2Options` controls `review_votes` (5), `enrich_votes` (3), `add_iterations` (3), +`include_examples`, `example_number`, `max_concurrency` (8), and `on_tool_error` +(`"skip"` by default, so one tool's failure does not abort the run). + +###### Feeding the code generator + +`specs_v2_to_v1` marks an item `skip=True` when today's codegen and runtime cannot +enforce it *correctly* — generated guards receive `args` + `api` only, with no acting +user and no conversation, and there is no post-invocation hook. So an item is skipped +when it has a `pending_for_user` gap, needs `message_history`, is `post_tool`, or reads +`system_vars`. Expect a v2 run over an identity-heavy policy to yield few guards; that +is the honest count of what can be enforced today, and each condition disappears as the +runtime gains the corresponding capability. + +> **Point v2 at its own `work_dir`.** v2 writes `.json`, the same filenames v1 +> uses, and v1's loader accepts a v2 file while ignoring its extra fields — which would +> leave every item `skip=False` and generate guards for rules that cannot be enforced. + #### Step 4: Generate Guard Code (Buildtime) ```python @@ -425,6 +517,14 @@ except Exception as e: - `generate_guards_code()`: Generate executable guard code from specifications - `LitellmModel`: LLM configuration for various providers +v2 specification generation (alternative to `generate_guard_specs`): + +- `generate_guard_specs_v2()`: Generate v2 specs, one per tool +- `generate_spec_conflicts_v2()`: Record conflicts across a complete spec set +- `generate_guard_specs_v2_full()`: Both of the above, in one call +- `specs_v2_to_v1()` / `spec_v2_to_v1()`: Convert v2 specs for `generate_guards_code()` +- `SpecV2Options`: Vote counts, iterations, concurrency, error policy + ### Runtime API - `load_toolguards()`: Load generated guards for runtime use diff --git a/docs/superpowers/evidence/2026-08-11-gen-spec-v2-test-baseline.md b/docs/superpowers/evidence/2026-08-11-gen-spec-v2-test-baseline.md new file mode 100644 index 0000000..1d00dca --- /dev/null +++ b/docs/superpowers/evidence/2026-08-11-gen-spec-v2-test-baseline.md @@ -0,0 +1,197 @@ +# gen_spec_v2 — test baseline + +Date: 2026-08-11 +Base commit: `e31b21c` (version 0.2.21), branch `sys_var`, nothing committed +LLM for e2e runs: `claude-sonnet-4-6` via azure +Design: [2026-08-10-gen-spec-v2-design.md](../specs/2026-08-10-gen-spec-v2-design.md) + +Written to be re-run and diffed after further benchmarks. Every number was measured, not +inferred; §7 records where a result came from an earlier state of the tree. + +## How to reproduce + +```bash +# No LLM needed. Definitive, ~53s. +PYTHONPATH=tests python -m pytest tests -q --ignore=tests/tmp --ignore=tests/buildtime/e2e + +# LLM needed. `PYTHONPATH=tests` is required for the `examples.*` imports. +PYTHONPATH=tests python -m pytest tests/buildtime/e2e/test_gen_spec_v2_codegen.py -q # 4 tests, ~4 min +PYTHONPATH=tests python -m pytest tests/buildtime/e2e/test_tau2_v2.py -q # 2 tests, ~4 min +PYTHONPATH=tests python -m pytest tests/buildtime/e2e/test_guard_set_delta.py -q -m delta # ~1.5 min + +# Types. gen_spec_v2 and all new tests must stay at 0. +python -m pyright src/toolguard/buildtime/gen_spec_v2 tests/buildtime/gen_spec_v2 +``` + +## Summary + +| Suite | Result | Time | +|---|---|---| +| Non-e2e (unit + contract) | **488 passed**, 0 failed, 3 pre-existing warnings | 52.9s | +| v2 e2e — calculator, 4 tool-input shapes | **4 passed** | in the 417s below | +| v2 e2e — tau2 `simple` | **passed** | " | +| v2 e2e — tau2 `complex_api` | **failed** — transient LLM invalid-JSON, §4 | " | +| v2 e2e combined (6 tests) | **5 passed, 1 failed** | 417.2s | +| v2 e2e — guard-set delta report | **1 passed** | 91.3s | +| v1 e2e — calculator (5) + tau2 (2) | not re-run this session; v1 source untouched | — | +| pyright — `gen_spec_v2` + all new tests | **0 errors** | | +| pyright — whole `src/toolguard` | 15 errors, **all pre-existing** in v1 modules | | + +Total: **493 automated checks passing, 1 failing**, the failure being a transport-level +model error rather than a logic defect. + +## 1. Non-e2e tests — 488 passed + +No LLM. This is the suite to trust for regressions. + +| File | Tests | Covers | +|---|---:|---| +| `test_refmatch_real_policy.py` | 96 | all 95 real ground-truth references ground back to themselves | +| `test_gen_py_contract.py` | 61 | python identifiers, generated-file collisions, what codegen receives | +| `test_adapter.py` | 41 | `skip` truth table, debug preservation, whole employee corpus | +| `test_stages.py` | 31 | create/expand/review/enrich/examples, incl. misshaped LLM responses | +| `test_serialize.py` | 30 | byte parity against all 28 ground-truth fixtures | +| `test_refmatch.py` | 22 | grounding: exact, wrapped, markdown-stripped, dash, snap, multi-segment, archive-when-ungrounded | +| `test_reconcile.py` | 19 | vote reconciliation, malformed votes | +| `test_pipeline.py` | 15 | 3 entry points, per-tool isolation, partial regeneration | +| `test_prompts.py` | 14 | every stage sees the inputs it judges against | +| `test_sysvars.py` | 13 | dict/path loading, `action_list`/`action_description` exclusion, nested values kept | +| `test_conflicts.py` | 12 | detection bounds, routing to each involved tool | +| `test_examples_only.py` | 8 | `generate_guard_examples_v2` touches only the examples | +| `test_context.py` | 5 | prompt-slice rendering | +| `test_tools_input.py` | 5 | callables / OpenAPI dict / `list[ToolInfo]` | +| **v2 subtotal** | **372** | | +| pre-existing suite | 116 | unchanged by this work | + +The 3 warnings are pre-existing: one `PytestCollectionWarning` for a test class with +`__init__`, two litellm coroutine warnings. + +## 2. v1 coverage parity + +Both e2e files import v1's own `assert_toolgurards_run`, so the two suites assert +identical enforcement (5 compliant calls + 5 violations) rather than similar-looking +enforcement. + +| v1 test | v2 counterpart | v2 status | +|---|---|---| +| `test_calculator.test_tool_functions_short` / `_long` | `test_gen_spec_v2_codegen.test_tool_functions` | pass | +| `test_calculator.test_tool_methods` | `..._codegen.test_tool_methods` | pass | +| `test_calculator.test_tools_langchain` | `..._codegen.test_tools_langchain` | pass | +| `test_calculator.test_tools_openapi_spec` | `..._codegen.test_tools_openapi_spec` | pass | +| `test_tau2.test_tau2_simple` | `test_tau2_v2.test_tau2_simple` | pass | +| `test_tau2.test_tau2_complex_api` | `test_tau2_v2.test_tau2_complex_api` | fail (§4) | +| `generate_guard_examples()` | `generate_guard_examples_v2()` | pass (8 unit tests) | + +Deliberately **not** ported: `PolicySpecOptions.spec_steps` phase selection. v2's stages +are not independent (enrich needs review's survivors; the adapter's `skip` needs enrich's +output), so arbitrary stage selection would silently emit specs whose `requires` and +`trigger` are empty rather than genuinely absent. + +## 3. What v2 produces on the employee corpus + +From the 28 ground-truth specs through the adapter: **15 of 75 items are codegen-able, +across 7 of 28 tools** — the pure-argument rules (salary positive, email domain, six-month +expiry, issue list[ToolInfo] (v1 is not touched) + refmatch.py reference grounding — rewritten, not ported + reconcile.py pure enrich-vote reconciliation + adapter.py SpecV2 -> runtime ToolGuardSpec + conflicts.py pairwise detection + routing + pipeline.py per-tool orchestration + entry points + stages/ create, expand, review, enrich, examples + prompts/ *.txt system prompts +``` + +Smith's prompts are Python modules that build both the system text and the user content. +v2 splits that: system text in `prompts/*.txt` (toolguard's `read_prompt_file` +convention), user-content assembly in `context.render_*`. + +`models`, `serialize`, `sysvars`, `refmatch`, `reconcile`, `adapter`, and conflict +routing are pure — no LLM, no I/O. The tests concentrate there. + +`gen_spec_v2/prompts` must be added to `[tool.hatch.build.targets.wheel.sources]` in +`pyproject.toml`, or the prompts do not ship in the wheel. + +## 1. Schema + +Pydantic models in `gen_spec_v2/models.py`. Buildtime-only: the runtime never sees them, +the adapter is the bridge. + +``` +Requires{system_vars: list[str], tool_history: list[ToolHistoryEntry] | None, + message_history: bool | None} +ToolHistoryEntry{tool: str, params: dict[str, str]} +PendingItem{type: PendingType, detail, question, suggested_tool?, suggested_source?} +Resolution{answer, decided_by, effect} +ResolvedItem = PendingItem + resolution +PolicyItemV2{id, name, description, compliance_examples, violation_examples, + references, trigger: Trigger, requires, pending_for_user, resolved_by_user} +Conflict{id, name, kind, conflicting_policies, description, question, resolution?} +SpecToolInfo{is_read_only: bool, user_enrichment: str} +SpecDebugV2{tool_info, archive: list[dict], notes: list | None} +SpecV2{tool_name, source_doc, policy_items, conflicts, debug} +``` + +`SpecToolInfo` avoids colliding with the existing `ToolInfo` (tool signatures, a +different concept); it still serializes under the key `tool_info`. + +Two enums where smith uses bare strings: + +- `Trigger = pre_tool | post_tool` +- `PendingType = missing_tool | missing_var | clarification` + +`PendingType` normalizes aliases on read (`missing_variable` → `missing_var`). This fixes +a live bug: smith's enrich prompt asks for `missing_var`, but two ground-truth specs +contain `missing_variable`, and smith's archive check only matches `missing_var` — so +those gaps were never recognized as blocking. Under v2's "keep the item" rule the +normalization is safe; it would have been load-bearing under smith's "archive" rule. + +### Serialization + +Pydantic emits declaration order, so `serialize.py` writes explicitly: + +``` +item: id, name, description, compliance_examples, violation_examples, + references, trigger, requires, [pending_for_user], [resolved_by_user] +spec: tool_name, source_doc, policy_items, [conflicts], debug +``` + +`[...]` is omitted when empty, as are `suggested_tool` / `suggested_source` when null and +`debug.notes` when absent. Output is `json.dumps(..., indent=2, ensure_ascii=False)` plus +a trailing newline — smith's exact bytes. + +## 2. Inputs + +```python +generate_guard_specs_v2( + policy_text: str, + tools: TOOLS | list[ToolInfo], + llm: I_TG_LLM, + work_dir: str | Path, + *, + tools2guard: list[str] | None = None, + system_vars: dict | str | Path | None = None, + source_doc: str = "", + options: SpecV2Options | None = None, +) -> list[SpecV2] +``` + +- **policy_text** — free-form. No bullet structure required. +- **tools** — callables, an OpenAPI dict, or a `list[ToolInfo]`. MCP servers arrive + through the existing `extra/mcp_tools_to_oas.py` path. `tools_input.py` implements the + `list[ToolInfo]` branch inside v2 so `gen_spec`'s `_tools_to_tool_infos` is not touched. +- **system_vars** — a dict, or a path to `sys_var.json`. Every top-level key is a subject + variable except `action_list` and `action_description`, which describe the agent's + tools rather than the acting user. + +`sys_var.json` value semantics, rendered into every stage's prompt: + +| Value shape | Rendered as | +|---|---| +| list — `"department": ["HR", ...]` | allowed values | +| scalar — `"user_id": 1` | example value | +| nested mapping or list — `"entitlements": {...}` | example value, rendered as-is | + +Each renders with its `input.extensions.subject.` path so the LLM names only +variables that exist. `requires.system_vars` is validated against those names; an +invented name is dropped and logged. + +`SpecV2Options`: `review_votes=5`, `enrich_votes=3`, `add_iterations=3`, +`include_examples=True`, `example_number=None`, `max_concurrency=8`, +`on_tool_error="skip" | "raise"`. + +## 3. Pipeline + +Per tool, in order: **create → expand ×N → review → enrich → examples**. + +| Stage | Does | +|---|---| +| `create` | Extracts the tool's policy items, plus `debug.tool_info{is_read_only, user_enrichment}`. Ids are assigned in code — `unique_id(tool, slugify(name), taken)` → `add_department.hr_only` — never by the LLM. | +| `expand` | `add_iterations` passes adding items the earlier passes missed. New items get ids the same way. | +| `review` | `review_votes` relevance votes per item; losers move to `debug.archive` as `{id, name, reason, stage}`. | +| `enrich` | `enrich_votes` votes per item, reconciled by a pure function. Produces `trigger`, `requires`, `pending_for_user`, and reference *validation*. | +| `examples` | Compliance / violation examples per item. | + +`enrich` is where the four alerts come from, in one vote payload: + +| Ask | Field | +|---|---| +| pre-tool vs post-tool | `trigger` | +| a tool call is needed first | `requires.tool_history` — with the params to call it with | +| the chat history is needed | `requires.message_history` | +| a missing tool / missing system var | `pending_for_user[type=missing_tool \| missing_var]` | +| a clarification question is needed | `pending_for_user[type=clarification]` | + +Reconciliation (`reconcile.py`, pure): majority `trigger` with ties → `pre_tool`; sorted +union of `system_vars`; the longest `tool_history` unless a strict majority say null; +`message_history` true on a strict majority; `pending_for_user` deduped by +`(type, question)`, first occurrence winning. + +Enrich only **validates** references — its result is intersected with the references the +item arrived with, so a vote can keep or drop but never add. If validation would blank a +grounded set, the original set is kept. + +Unlike smith, an item with a `missing_tool` / `missing_var` gap and no other enforcement +source is **kept**, carrying its alert. That is the point of the feature. Nothing +downstream is endangered: the adapter marks anything with `pending_for_user` as skipped. + +Per-tool failures are isolated under `asyncio.Semaphore(max_concurrency)`; +`on_tool_error="skip"` records the failure and lets the other tools finish. + +## 4. Entry points + +| Function | Does | +|---|---| +| `generate_guard_specs_v2` | Per-tool specs → `.json`. No cross-tool work. | +| `generate_spec_conflicts_v2` | Pairwise within-tool conflict detection over a spec set — passed in, or every `.json` loaded from `work_dir`. Rewrites specs whose `conflicts` changed. | +| `generate_guard_specs_v2_full` | All tools, then conflicts. One call for a full build. | + +Conflict detection is within-tool pairwise upper-triangle (item `i` vs items `i+1..n`), +which keeps every prompt bounded by one tool's item count. A conflict can still name ids +from other tools, so routing handles both cases: + +- all `conflicting_policies` share one tool prefix → attach to that tool's spec +- prefixes span several tools → attach to **each** involved tool's spec, deduped by id, + so every spec stays self-contained (smith sent these to `global.json`, which v2 does + not have) + +## 5. Reference grounding + +v1's `find_mismatched_references` has five concrete defects: + +1. `normalize_text` only lowercases, so a reference differing by line wrap, `**bold**`, + or an em-dash versus a hyphen never matches. +2. `end_idx = start_idx + len(reference)` projects a normalized match back using the + *reference's* length — correct only while normalization is length-preserving. Any real + normalization silently yields truncated or over-long quotes. +3. The fallback splits a reference in two and accepts it if each half appears *anywhere*; + a half can be one word matched in an unrelated section. +4. No fuzzy matching — the `difflib` attempt is commented out. +5. `unmatched_policies` is computed, returned, and discarded by the caller. No signal. + +`refmatch.py` replaces it. Still fuzzy — no bullet requirement: + +- `normalize(text) -> (str, offsets)` — NFKC, casefold, unify dashes and quotes, strip + markdown emphasis and backticks, collapse whitespace, **with a normalized→original + offset map** so every match projects back to an exact original substring regardless of + length change. +- `segments(doc) -> list[Span]` — candidate units with original spans: bullet lines where + present, otherwise sentences within paragraphs. Boundaries are preferred, never + required. +- `ground(reference, doc)` — in order: exact normalized substring, snapped outward to the + enclosing segment when the match covers most of it; else the best segment by + `difflib.SequenceMatcher` ratio above a named threshold (stdlib, no new dependency); + else 2+ **consecutive** segments covering the reference, replacing the arbitrary + two-part split; else `None`. +- `ground_spec(spec, policy_text)` — rewrites references to grounded verbatim spans, + dedupes, preserves order. An ungrounded reference is kept as written **and** recorded in + `debug` with a warning, so nothing silently passes a paraphrase off as a quote. + +Prompt side: create and expand ask for verbatim contiguous quotes, one per rule, no +ellipsis, no paraphrase. + +## 6. Adapter and `gen_py` compatibility + +`adapter.spec_v2_to_v1(spec) -> ToolGuardSpec`, pure: + +```python +skip = (bool(item.pending_for_user) + or bool(item.requires.message_history) + or item.trigger == Trigger.post_tool + or bool(item.requires.system_vars)) +``` + +Each condition marks something today's codegen and runtime cannot enforce *correctly* — +generated guards receive `args` + `api` only, with no subject and no message history, and +there is no post-invocation hook. A guard generated anyway would be wrong enforcement +rather than absent enforcement. Each condition drops out as the runtime gains that +capability. + +`id`, `trigger`, and `requires` ride along in `item.debug`; `source_doc`, `conflicts`, +`tool_info`, and `archive` in `spec.debug`. + +`gen_py` consumes exactly five fields, which pins the adapter's contract: + +| Field | Used for | Adapter consequence | +|---|---|---| +| `spec.tool_name` | guard module + function name | Must not emit specs for tools absent from `tools` | +| `item.name` | python module, function, and test-file names (`.` → `_` in place) | v2 names are human sentences like v1's. Two items in one tool sharing a name would collide on one file — `unique_id` dedupes ids, not names — so the adapter disambiguates a colliding name with the id suffix | +| `item.description` | the pseudo-code prompt that becomes the guard body | unchanged | +| `item.compliance_examples` / `violation_examples` | test generation | unchanged | +| `item.skip` | `[i for i in spec.policy_items if not i.skip]`, then specs with zero items are dropped | the filter above | + +Two consequences to expect: + +- **Most employee-example tools produce no guards.** When every rule for a tool is + system-var-driven, all its items are skipped and `gen_toolguards` drops the spec. That + is correct. The "adapter drives gen_py" test therefore runs on calculator/tau2, where + items survive; the employee example is used to assert the *skip classification*. +- `update_employee.home_address_same_country` needs `tool_history: [get_employee]` but no + system vars, no chat history, and is `pre_tool` — so it is **not** skipped and does get + codegen'd. `tool_dependencies.py` then re-infers `get_employee` with an LLM call even + though `requires.tool_history` states it. Harmless now; the most obvious first win when + `gen_py` learns v2 natively. + +### Known compatibility caveat + +v2 writes `.json` — the same filenames v1 uses. Pydantic's default +`extra='ignore'` means v1's `ToolGuardSpec.load` *accepts* a v2 file silently, dropping +`id` / `trigger` / `requires` / `pending_for_user` and yielding every item `skip=False`. +Since `gen_toolguards` filters on `skip`, a caller that globs a shared spec directory +would generate wrong-enforcement guards. Documented, not fixed: point v2 at its own +`work_dir`, which is what the docs and tests do. + +## 7. Testing + +| Layer | Test | LLM | +|---|---|---| +| Serializer | Each ground-truth fixture: load → dump → identical bytes (modulo `missing_variable` → `missing_var`) | no | +| Models | Enum aliasing; optional keys absent/present; tolerant read | no | +| `sysvars` | dict and path inputs; list → allowed values, scalar → example value | no | +| `refmatch` | Exact; line-wrapped; markdown-stripped; em-dash vs hyphen; mid-sentence snap; two consecutive segments; no match; offset projection under length-changing normalization | no | +| `reconcile` | Tie → `pre_tool`; union of `system_vars`; majority-null `tool_history`; `message_history` majority; pending dedupe | no | +| `adapter` | `skip` truth table; name-collision disambiguation; output loads through `ToolGuardSpec.load` | no | +| Conflicts | Single-prefix → that tool; multi-prefix → each involved tool | no | +| Pipeline | Fake LLM: per-tool isolation, `on_tool_error`, partial regeneration leaves other files untouched, conflicts entry point loads from disk | no | +| e2e | v2 → adapter → `generate_guards_code`: guards compile and their generated tests pass (calculator) | yes | +| e2e | Employee policy + MCP server → shape parity with ground truth: every tool file present, ids well-formed, trigger/requires on every item, pending types in the enum | yes | + +Ground-truth fixtures are copied into `tests/data/specs_v2/` to avoid a cross-repo test +dependency. + +## Implementation notes + +Decisions taken while building, beyond what the design above specified. + +**Two non-subject keys are excluded by name; every value shape is kept.** The design +said toolguard applies no filtering and the caller passes a clean dict. The real +`system_vars.json` carries `action_list` (34 tool names) and `action_description` (a +35-entry mapping) alongside the four subject variables, and rendering those into every +prompt is noise — they describe the agent's own tools, not the acting user. +`load_system_vars` drops exactly those two keys (`NON_SUBJECT_KEYS`). + +`sys_var.json` is **not** assumed to be flat: a nested mapping or list is kept and +rendered as a subject variable, since a structured attribute of the acting user is +still an attribute of the acting user. A list renders as allowed values; anything else +renders as an example value. + +**Ungrounded references are recorded in `debug.notes`.** `notes` is +presence-tracked and omitted when absent, so a spec whose every reference grounded is +byte-identical to smith's output; only a spec with a quote that could not be located +carries the extra key. + +**Enrich validates against reality, not just across votes.** Beyond reconciliation, an +undeclared system variable and a `tool_history` entry naming a tool that does not exist +are both dropped with a warning. Either would otherwise compile into a guard reading +something that is not there. + +**Conflicts naming no known policy item are dropped**, and `attach_conflicts` replaces +rather than appends, so rerunning the conflicts pass is idempotent. + +**Adapter name disambiguation happens in codegen's namespace.** Comparing raw names was +not enough: `gen_py` rewrites `.` to `_` and then snake-cases, so "rule v1.0" and +"rule v1_0" are different names that land in the same generated file. The adapter now +keys collisions on `to_py_module_name(f"guard_{name.replace('.', '_')}")`. The +disambiguating suffix is appended with a space and no punctuation, because +`to_snake_case` leaves punctuation in place. + +**A pre-existing v1 limitation, found and not fixed:** `to_snake_case` does not strip +parentheses, so an item named "Flight booking passenger limit (foreign-domain rule)" +yields `guard_flight_booking_passenger_limit_(foreign_domain_rule)` — not a valid +python identifier. It bites only unskipped items, and every parenthesized name in the +employee corpus is skipped, so nothing breaks today. Fixing `py.to_snake_case` was out +of scope ("no change to v1"); the adapter's own suffix is punctuation-free so it cannot +walk into the same hole. + +**`on_tool_error` defaults to `skip`**, so one tool's failure never costs a whole run. +Every requested tool gets a spec file even when no rule governs it: an empty spec is a +real answer, and its absence would be indistinguishable from a failed run. + +### Over-attachment: found by the tau2 port, fixed in the prompts + +The first tau2 run exposed a real defect. On the one-sentence policy "Users cannot book a +flight for more than 5 passengers", v1 binds the rule to two tools; v2 bound it to five, +adding `update_reservation_baggages`, `update_reservation_flights`, and +`transfer_to_human_agents`. On the cancellation policy, v2 additionally bound rules to +`book_reservation`, `get_reservation_details`, and `get_flight_status`. + +These were not hallucinations — each cited the correct policy line — but they were bound +to tools the policy does not govern, and they were **not** skipped by the adapter, so +guards were generated: four guarded tools instead of two, including one on +`get_reservation_details`, a read-only lookup. A false guard on an unrelated tool is worse +than a missing guard: it denies legitimate calls. + +The cause was in the prompts. `review.txt` said "do NOT reject an item merely because the +same rule also applies to other tools", which is right for genuine cross-cutting rules +(confirm-before-write applies to every write tool) but was read as licence to bind any +rule to any tool touching the same record. + +The fix is one criterion, added to `create.txt`, `expand.txt`, and `review.txt`: + +> Is there some set of arguments for which calling THIS tool performs the action the rule +> restricts, or produces the state the rule forbids? + +It separates the two cases cleanly. `update_reservation_passengers` can set six +passengers, so the cap governs it; `update_reservation_baggages` cannot change the +passenger count under any arguments, so it does not. A write tool does perform the write +that confirm-before-write restricts, so cross-cutting rules still bind. `create.txt` +additionally requires each `description` to name both the action of *this* tool being +restricted and the exact value tested, and to omit the item when it cannot. + +Sharpening the prompts took tau2 `simple` from five governed tools to three and +`complex_api` from five to one (correct), but did not fully close it. +`update_reservation_flights` still binds the passenger cap, and its own description says +why it should not: *"The passenger count is not a direct parameter of this tool, but can +be verified by calling get_reservation_details"*. The model stated the criterion and +overrode it, so prompt wording has reached its limit there. The tau2 runs therefore use +`review_votes=5` (the default) rather than the calculator e2e's 3: with 40-odd tools there +are many chances to mis-bind, and the relevance vote is what has to reject them. + +If votes prove insufficient, the next lever is verification in code rather than more +wording: have create/expand also return **which value** each rule constrains, then check +that the value is a parameter of this tool or a field of its result and drop the item +otherwise. That is the pattern v2 already uses for `system_vars` and `tool_history` — the +model proposes, code verifies. It has to be skipped when a rule constrains an *action* +rather than a value ("confirm before writing" names no parameter), and it correctly keeps +`update_employee.home_address_same_country`, whose `country_code` *is* a parameter. + +### Items grounded in nothing are archived + +A second defect from the same run: an item cited text from the `AirlineTools` docstring +rather than the policy document. `refmatch` detected it but v2 kept the item, so a rule +nobody wrote reached the spec. + +`ground_spec` now archives an item when **none** of its references can be located in the +policy document, with `stage: "refmatch"` and the reason. An item that keeps at least one +grounded quote still survives with its ungrounded ones intact and recorded in +`debug.notes` — dropping those would weaken an otherwise sound item — and an item that +arrived with no references at all is left alone, since `create`/`expand` already refuse to +emit those. + +### Where the tests live + +| File | Covers | LLM | +|---|---|---| +| `tests/buildtime/gen_spec_v2/test_serialize.py` | byte parity over all 28 fixtures | no | +| `test_sysvars.py` | dict/path loading, shape filtering, rendering, name validation | no | +| `test_tools_input.py` | callables / OpenAPI dict / `list[ToolInfo]` | no | +| `test_context.py` | prompt-slice rendering | no | +| `test_refmatch.py` | grounding, synthetic cases per v1 defect | no | +| `test_refmatch_real_policy.py` | all 95 real references ground to themselves | no | +| `test_reconcile.py` | vote reconciliation, malformed votes | no | +| `test_adapter.py` | `skip` truth table, debug preservation, real corpus | no | +| `test_prompts.py` | every stage sees the inputs it judges against | no | +| `test_stages.py` | the five stages, including misshaped responses | no | +| `test_conflicts.py` | detection bounds, routing to each involved tool | no | +| `test_pipeline.py` | three entry points, per-tool isolation, partial regeneration | no | +| `test_gen_py_contract.py` | identifiers, file collisions, what codegen receives | no | +| `test_examples_only.py` | `generate_guard_examples_v2` rewrites only the examples | no | +| `tests/buildtime/e2e/test_gen_spec_v2_codegen.py` | v2 → adapter → guards that run, for all four tool-input shapes | yes | +| `tests/buildtime/e2e/test_tau2_v2.py` | tau2 airline, simple + complex_api | yes | +| `tests/buildtime/e2e/test_guard_set_delta.py` | reports the v1-vs-v2 guard-set difference (`-m delta`) | yes | + +### v1 coverage parity + +The e2e files mirror v1's variants one-for-one, and both import v1's +`assert_toolgurards_run`, so the two suites assert identical enforcement rather than +similar-looking enforcement: + +| v1 test | v2 counterpart | +|---|---| +| `test_calculator.test_tool_functions_short` / `_long` | `test_gen_spec_v2_codegen.test_tool_functions` | +| `test_calculator.test_tool_methods` | `test_gen_spec_v2_codegen.test_tool_methods` | +| `test_calculator.test_tools_langchain` | `test_gen_spec_v2_codegen.test_tools_langchain` | +| `test_calculator.test_tools_openapi_spec` | `test_gen_spec_v2_codegen.test_tools_openapi_spec` | +| `test_tau2.test_tau2_simple` | `test_tau2_v2.test_tau2_simple` | +| `test_tau2.test_tau2_complex_api` | `test_tau2_v2.test_tau2_complex_api` | +| `generate_guard_examples()` | `generate_guard_examples_v2()` + `test_examples_only.py` | + +Two deliberate differences in the tau2 port: item counts are asserted as `>= 1` rather +than `== 1`, because v2 runs an `expand` stage v1's short-options run does not and a +legitimate extra rule must not fail the test; and `complex_api` additionally asserts +that the cancellation rule declares a `tool_history` requirement, which v1's schema +cannot express. + +v1's `PolicySpecOptions.spec_steps` phase selection has no v2 equivalent — `SpecV2Options` +exposes `include_examples` and vote counts but not arbitrary stage selection. That is a +deliberate simplification, not an oversight: v2's stages are not independent (triage +depends on enrich, which depends on review's survivors). + +The e2e file calls `load_dotenv()` at import, because conftest's autouse fixture loads +`.env` after collection and a `skipif` is evaluated during it. + +## Non-goals + +`global.json` and orphan-rule items; the rule→items coverage report; a v1→v2 upgrade +function; persisting a user's `skip` decision in a v2 file; a runtime post-invocation +hook; a subject or message-history parameter in generated guards; replacing +`tool_dependencies.py`'s inference with `requires.tool_history`; any change to v1. diff --git a/pyproject.toml b/pyproject.toml index 0654016..a85fb2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ packages = ["src/toolguard"] [tool.hatch.build.targets.wheel.sources] "src/toolguard/buildtime/gen_spec/prompts" = "toolguard/buildtime/gen_spec/prompts" +"src/toolguard/buildtime/gen_spec_v2/prompts" = "toolguard/buildtime/gen_spec_v2/prompts" "src/toolguard/buildtime/gen_py/templates" = "toolguard/buildtime/gen_py/templates" [tool.hatch.build] @@ -70,3 +71,7 @@ filterwarnings = [ ] asyncio_mode = "auto" + +markers = [ + "delta: opt-in report comparing v1 and v2 guard sets (two full LLM spec runs)", +] diff --git a/src/toolguard/buildtime/__init__.py b/src/toolguard/buildtime/__init__.py index b9794cb..88d4eff 100644 --- a/src/toolguard/buildtime/__init__.py +++ b/src/toolguard/buildtime/__init__.py @@ -10,6 +10,17 @@ PolicySpecOptions, PolicySpecStep, ) +from toolguard.buildtime.gen_spec.data_types import ToolInfo +from toolguard.buildtime.gen_spec_v2 import ( + SpecV2, + SpecV2Options, + generate_guard_examples_v2, + generate_guard_specs_v2, + generate_guard_specs_v2_full, + generate_spec_conflicts_v2, + spec_v2_to_v1, + specs_v2_to_v1, +) __all__ = [ "generate_guard_specs", @@ -21,6 +32,16 @@ "ToolGuardSpec", "ToolGuardsCodeGenerationResult", "TOOLS", + "ToolInfo", "PolicySpecOptions", "PolicySpecStep", + # v2 spec generation (alternative to generate_guard_specs) + "generate_guard_specs_v2", + "generate_spec_conflicts_v2", + "generate_guard_specs_v2_full", + "generate_guard_examples_v2", + "SpecV2", + "SpecV2Options", + "spec_v2_to_v1", + "specs_v2_to_v1", ] diff --git a/src/toolguard/buildtime/buildtime.py b/src/toolguard/buildtime/buildtime.py index b155abc..7f339f4 100644 --- a/src/toolguard/buildtime/buildtime.py +++ b/src/toolguard/buildtime/buildtime.py @@ -14,6 +14,15 @@ ToolGuardSpecGenerator, _tools_to_tool_infos, ) +from toolguard.buildtime.gen_spec_v2 import ( # noqa: F401 - re-exported API + SpecV2, + SpecV2Options, + generate_guard_specs_v2, + generate_guard_specs_v2_full, + generate_spec_conflicts_v2, + spec_v2_to_v1, + specs_v2_to_v1, +) from toolguard.buildtime.llm import I_TG_LLM from toolguard.buildtime.utils.open_api import OpenAPI from toolguard.runtime.data_types import ToolGuardsCodeGenerationResult, ToolGuardSpec diff --git a/src/toolguard/buildtime/gen_spec_v2/__init__.py b/src/toolguard/buildtime/gen_spec_v2/__init__.py new file mode 100644 index 0000000..80c505c --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/__init__.py @@ -0,0 +1,85 @@ +"""Guard-spec generation, v2. + +A second, parallel spec generator. Where v1 emits a rule's text and examples, +v2 also records who is acting (``requires.system_vars``), when the rule applies +(``trigger``), what else is needed to decide it (``requires.tool_history``, +``requires.message_history``), and what is missing to enforce it at all +(``pending_for_user``). + +Lives beside ``gen_spec`` (v1), which is unchanged and remains the default. The +code generator and runtime still speak v1; :func:`spec_v2_to_v1` is the bridge. + +Typical use:: + + specs = await generate_guard_specs_v2_full( + policy_text, tools, llm, work_dir, + system_vars="sys_var.json", source_doc="policy.md", + ) + v1_specs = specs_v2_to_v1(specs, known_tools=[t.name for t in tools]) +""" + +from toolguard.buildtime.gen_spec_v2.adapter import ( + is_enforceable_today, + spec_v2_to_v1, + specs_v2_to_v1, +) +from toolguard.buildtime.gen_spec_v2.models import ( + Conflict, + PendingItem, + PendingType, + PolicyItemV2, + Requires, + Resolution, + ResolvedItem, + SpecDebugV2, + SpecToolInfo, + SpecV2, + ToolHistoryEntry, + Trigger, +) +from toolguard.buildtime.gen_spec_v2.pipeline import ( + SpecV2Options, + generate_guard_examples_v2, + generate_guard_specs_v2, + generate_guard_specs_v2_full, + generate_spec_conflicts_v2, +) +from toolguard.buildtime.gen_spec_v2.serialize import ( + dump_spec, + load_spec, + load_specs, + spec_from_dict, + spec_to_dict, +) + +__all__ = [ + # entry points + "generate_guard_specs_v2", + "generate_spec_conflicts_v2", + "generate_guard_specs_v2_full", + "generate_guard_examples_v2", + "SpecV2Options", + # adapter to what codegen and the runtime consume + "spec_v2_to_v1", + "specs_v2_to_v1", + "is_enforceable_today", + # schema + "SpecV2", + "PolicyItemV2", + "Requires", + "ToolHistoryEntry", + "PendingItem", + "PendingType", + "Resolution", + "ResolvedItem", + "Conflict", + "SpecToolInfo", + "SpecDebugV2", + "Trigger", + # serialization + "load_spec", + "load_specs", + "dump_spec", + "spec_to_dict", + "spec_from_dict", +] diff --git a/src/toolguard/buildtime/gen_spec_v2/adapter.py b/src/toolguard/buildtime/gen_spec_v2/adapter.py new file mode 100644 index 0000000..474166b --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/adapter.py @@ -0,0 +1,147 @@ +"""Converting v2 specs into the v1 specs that codegen consumes. + +``gen_py`` and the runtime speak v1. This module is the only bridge, and its +whole job is honesty about what today's pipeline can enforce: a generated +guard receives ``args`` and ``api`` — no acting user, no chat history — and +the runtime has no post-invocation hook. An item depending on any of those +is marked ``skip`` so codegen leaves it alone, rather than emitting a guard +that checks the wrong thing. + +Each condition in :func:`is_enforceable_today` drops out as the runtime gains +the corresponding capability. +""" + +from typing import Iterable, List, Optional + +from loguru import logger + +from toolguard.buildtime.gen_spec_v2.models import PolicyItemV2, SpecV2, Trigger +from toolguard.buildtime.utils import py +from toolguard.runtime.data_types import ToolGuardSpec, ToolGuardSpecItem + + +def is_enforceable_today(item: PolicyItemV2) -> bool: + """Whether v1 codegen can generate a correct guard for ``item``. + + False when the item needs the acting user's identity, the conversation, + evaluation after the call, or an answer from a human. ``tool_history`` + alone is fine: generated guards already get an ``api`` handle. + """ + return not ( + item.pending_for_user + or item.requires.message_history + or item.trigger == Trigger.post_tool + or item.requires.system_vars + ) + + +def _unenforceable_reason(item: PolicyItemV2) -> str: + if item.pending_for_user: + types = ", ".join(sorted({p.type.value for p in item.pending_for_user})) + return f"pending_for_user ({types})" + if item.requires.message_history: + return "requires message history" + if item.trigger == Trigger.post_tool: + return "post_tool trigger" + return f"requires system vars ({', '.join(item.requires.system_vars)})" + + +def _module_key(name: str) -> str: + """The generated module name ``gen_py`` would derive from ``name``. + + Collisions have to be judged here, not on the raw name: codegen rewrites + ``.`` to ``_`` and then snake-cases, so "rule v1.0" and "rule v1_0" are + distinct names that would land in the same file. + """ + return py.to_py_module_name(f"guard_{name.replace('.', '_')}") + + +def _unique_names(items: Iterable[PolicyItemV2]) -> List[str]: + """Item names, disambiguated so each maps to its own generated file. + + ``gen_py`` derives a module, a guard function, and a test file from + ``item.name``. Ids are unique but names are not, so a name that would + collide gains the distinguishing part of its id. The suffix is appended + with a space and no punctuation, because snake-casing leaves punctuation + in place and would produce an invalid identifier. + """ + names: List[str] = [] + used_keys = set() + for item in items: + name = item.name + if _module_key(name) in used_keys: + suffix = item.id.split(".", 1)[-1] + name = f"{item.name} {suffix}" + n = 2 + while _module_key(name) in used_keys: + name = f"{item.name} {suffix} {n}" + n += 1 + names.append(name) + used_keys.add(_module_key(name)) + return names + + +def spec_v2_to_v1(spec: SpecV2) -> ToolGuardSpec: + """Convert one v2 spec, deriving ``skip`` and preserving v2 fields in ``debug``.""" + names = _unique_names(spec.policy_items) + items = [] + + for item, name in zip(spec.policy_items, names): + enforceable = is_enforceable_today(item) + if not enforceable: + logger.debug( + "{}: skipped for codegen — {}", item.id, _unenforceable_reason(item) + ) + items.append( + ToolGuardSpecItem( + name=name, + description=item.description, + references=list(item.references), + compliance_examples=list(item.compliance_examples), + violation_examples=list(item.violation_examples), + skip=not enforceable, + debug={ + "id": item.id, + "trigger": item.trigger.value, + "requires": item.requires.model_dump(), + **( + {"skip_reason": _unenforceable_reason(item)} + if not enforceable + else {} + ), + }, + ) + ) + + return ToolGuardSpec( + tool_name=spec.tool_name, + policy_items=items, + debug={ + "source_doc": spec.source_doc, + "tool_info": spec.debug.tool_info.model_dump(), + "archive": list(spec.debug.archive), + "conflicts": [c.model_dump() for c in spec.conflicts], + }, + ) + + +def specs_v2_to_v1( + specs: Iterable[SpecV2], known_tools: Optional[Iterable[str]] = None +) -> List[ToolGuardSpec]: + """Convert many specs, optionally dropping those for tools that don't exist. + + Passing ``known_tools`` guards against handing codegen a spec whose + ``tool_name`` has no tool behind it, which would try to generate a guard + for nothing. + """ + allowed = set(known_tools) if known_tools is not None else None + converted = [] + for spec in specs: + if allowed is not None and spec.tool_name not in allowed: + logger.warning( + "Dropping spec '{}': no such tool in the supplied tool set", + spec.tool_name, + ) + continue + converted.append(spec_v2_to_v1(spec)) + return converted diff --git a/src/toolguard/buildtime/gen_spec_v2/conflicts.py b/src/toolguard/buildtime/gen_spec_v2/conflicts.py new file mode 100644 index 0000000..f6153bb --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/conflicts.py @@ -0,0 +1,118 @@ +"""Finding policy items that disagree, and recording the question that settles it. + +All of a tool's policy items must hold together, so a permission granted by one +and a prohibition written by another silently resolve to "denied". That is +often not what the policy author intended, and it is invisible in the items +themselves — hence a separate pass that reports the pair and the question a +human needs to answer. + +Detection is pairwise *within* a tool (item ``i`` against items ``i+1..n``), +which keeps every prompt bounded by one tool's item count instead of the whole +spec set. A conflict may still name items from other tools, and +:func:`attach_conflicts` routes those to every tool involved: with no +``global.json`` in v2, each spec has to carry its own conflicts to stay +self-contained. +""" + +import asyncio +from typing import Any, Dict, Iterable, List, Sequence + +from loguru import logger + +from toolguard.buildtime.gen_spec_v2 import prompts +from toolguard.buildtime.gen_spec_v2.models import Conflict, SpecV2 +from toolguard.buildtime.gen_spec_v2.stages._shared import dict_list, messages +from toolguard.buildtime.llm import I_TG_LLM + + +def _as_dict(item) -> Dict[str, Any]: + return {"id": item.id, "name": item.name, "description": item.description} + + +async def _tool_conflicts(llm: I_TG_LLM, spec: SpecV2) -> List[Dict[str, Any]]: + items = [_as_dict(item) for item in spec.policy_items] + if len(items) < 2: + return [] + + system = prompts.system("conflicts") + calls = [ + llm.chat_json( + messages( + system, + prompts.conflicts_user(spec.tool_name, items[i], items[i + 1 :]), + ) + ) + for i in range(len(items) - 1) + ] + + responses = await asyncio.gather(*calls) + raw: List[Dict[str, Any]] = [] + for response in responses: + raw.extend(dict_list(response, "conflicts")) + return raw + + +async def find_conflicts(llm: I_TG_LLM, specs: Sequence[SpecV2]) -> List[Conflict]: + """Detect conflicts across ``specs``, deduped by id, first-seen winning. + + A malformed entry is skipped rather than raised: one bad sub-response must + not cost the run every other conflict it found. A conflict naming no known + policy item is also dropped — there is nothing a human could act on. + """ + known_items = {item.id for spec in specs for item in spec.policy_items} + + results = await asyncio.gather(*[_tool_conflicts(llm, spec) for spec in specs]) + + conflicts: List[Conflict] = [] + seen = set() + for raw_conflicts in results: + for raw in raw_conflicts: + try: + conflict = Conflict.model_validate({**raw, "resolution": None}) + except Exception: # noqa: BLE001 - one bad entry must not lose the rest + logger.warning("Skipping malformed conflict entry: {}", raw) + continue + + if conflict.id in seen: + continue + if not any(p in known_items for p in conflict.conflicting_policies): + logger.warning( + "Dropping conflict '{}': names no known policy item", conflict.id + ) + continue + + seen.add(conflict.id) + conflicts.append(conflict) + + return conflicts + + +def _tools_of(conflict: Conflict) -> Iterable[str]: + return { + policy_id.split(".", 1)[0] + for policy_id in conflict.conflicting_policies + if "." in policy_id + } + + +def attach_conflicts(specs: Sequence[SpecV2], conflicts: Sequence[Conflict]) -> None: + """Replace each spec's ``conflicts`` with the ones that involve it, in place. + + Replacing rather than appending makes a rerun idempotent. A conflict + spanning several tools is attached to each of them, so no signal is lost + now that there is no shared spec to hold it. + """ + by_tool = {spec.tool_name: spec for spec in specs} + for spec in specs: + spec.conflicts = [] + + for conflict in conflicts: + involved = [tool for tool in _tools_of(conflict) if tool in by_tool] + if not involved: + logger.warning( + "Dropping conflict '{}': none of its tools are in this spec set", + conflict.id, + ) + continue + for tool in sorted(involved): + by_tool[tool].conflicts.append(conflict) diff --git a/src/toolguard/buildtime/gen_spec_v2/context.py b/src/toolguard/buildtime/gen_spec_v2/context.py new file mode 100644 index 0000000..db9bd2c --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/context.py @@ -0,0 +1,98 @@ +"""The shared per-run context every stage renders its prompt slice from. + +Each stage needs a different combination of the policy document, the tool +catalog, the current tool's detail, and the system variables. Building them +all once and rendering slices from pure helpers keeps every stage looking at +the same inputs — in v1 the feasibility reviewer was never shown the system +variables it was implicitly judging against. +""" + +import json +from typing import List, Sequence + +from pydantic import BaseModel, Field + +from toolguard.buildtime.gen_spec.data_types import ToolInfo +from toolguard.buildtime.gen_spec_v2.sysvars import ( + SystemVars, + SystemVarsInput, + load_system_vars, + render_system_vars, +) +from toolguard.buildtime.gen_spec_v2.tools_input import TOOLS_V2, to_tool_infos + + +class GenContext(BaseModel): + """Everything generation knows, independent of which tool is in hand.""" + + policy_text: str + tools: List[ToolInfo] = Field(default_factory=list) + system_vars: SystemVars = Field(default_factory=SystemVars) + + @classmethod + def build( + cls, + policy_text: str, + tools: TOOLS_V2, + system_vars: SystemVarsInput = None, + ) -> "GenContext": + return cls( + policy_text=policy_text, + tools=to_tool_infos(tools), + system_vars=load_system_vars(system_vars), + ) + + def tool_names(self) -> List[str]: + return [tool.name for tool in self.tools] + + def tool_by_name(self, name: str) -> ToolInfo: + for tool in self.tools: + if tool.name == name: + return tool + raise KeyError(f"Unknown tool: {name}") + + def render_policy(self) -> str: + """The document verbatim. + + v2 does not split it into rules, so headings and section order — real + context for what a rule applies to — reach the model intact, and every + reference the model quotes is a span of this exact text. + """ + return self.policy_text + + def render_system_vars(self) -> str: + return render_system_vars(self.system_vars) + + def render_tools_overview(self) -> str: + """A compact ``- : `` catalog of every tool.""" + if not self.tools: + return "(no tools available)" + return "\n".join(f"- {tool.name}: {tool.description}" for tool in self.tools) + + def render_tool_detail(self, tool: ToolInfo) -> str: + """The current tool's signature and full parameter detail.""" + params = { + name: { + "type": param.type, + "description": param.description, + "required": param.required, + } + for name, param in tool.parameters.items() + } + return ( + f"Tool name: {tool.name}\n" + f"Tool description: {tool.description}\n" + f"Tool signature: {tool.signature}\n" + f"Tool parameters:\n{json.dumps(params, indent=2)}" + ) + + def render_other_tools(self, tool: ToolInfo) -> str: + """The catalog minus the current tool. + + What a rule can lean on for a prior lookup is exactly the *other* + tools, so ``requires.tool_history`` is judged against this list. + """ + others: Sequence[ToolInfo] = [t for t in self.tools if t.name != tool.name] + if not others: + return "(no other tools available)" + return "\n".join(f"- {t.name}: {t.description}" for t in others) diff --git a/src/toolguard/buildtime/gen_spec_v2/models.py b/src/toolguard/buildtime/gen_spec_v2/models.py new file mode 100644 index 0000000..afd9ec4 --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/models.py @@ -0,0 +1,201 @@ +"""The v2 guard-spec schema. + +These models describe the on-disk JSON that ``gen_spec_v2`` produces: the +v1 spec (``toolguard.runtime.data_types.ToolGuardSpec``) plus everything +needed to say *who* is acting, *when* a rule applies, *what else* is needed +to decide it, and *what is missing* to enforce it at all. + +They are buildtime-only. The runtime and the code generator consume v1 +specs; :mod:`toolguard.buildtime.gen_spec_v2.adapter` is the bridge. + +Serialization lives in :mod:`toolguard.buildtime.gen_spec_v2.serialize`, not +here: the on-disk key order is fixed by the format and differs from these +classes' declaration order. +""" + +import re +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, Field + +from toolguard.buildtime.compat.strenum import StrEnum + + +def slugify(text: str) -> str: + """Lowercase, collapse non-alphanumeric runs to a single ``_``, strip edges.""" + return re.sub(r"[^a-z0-9]+", "_", text.lower()).strip("_") + + +def unique_id(tool: str, slug: str, taken: set) -> str: + """Return ``f"{tool}.{slug}"``, suffixing ``_2``, ``_3``, ... until unused. + + ``taken`` is read-only; it is never mutated here. + """ + base = f"{tool}.{slug}" + if base not in taken: + return base + n = 2 + while f"{base}_{n}" in taken: + n += 1 + return f"{base}_{n}" + + +class Trigger(StrEnum): + """When a policy item is evaluated.""" + + pre_tool = "pre_tool" + """Before the tool is invoked, against its arguments.""" + + post_tool = "post_tool" + """After the tool returns, against its result.""" + + +class PendingType(StrEnum): + """Why a policy item cannot be enforced without a human's input.""" + + missing_tool = "missing_tool" + """The rule needs a tool that the tool set does not expose.""" + + missing_var = "missing_var" + """The rule needs a subject variable that ``system_vars`` does not supply.""" + + clarification = "clarification" + """The rule is ambiguous; only a human can pick the deterministic reading.""" + + @classmethod + def _missing_(cls, value: object) -> Optional["PendingType"]: + """Accept known spellings that drift out of generation. + + ``missing_variable`` appears in specs generated before the vocabulary + was an enum, and is silently ignored by any code matching on + ``missing_var``. Normalize instead of rejecting, so old specs load. + """ + # Dict[str, Any], not Dict[str, PendingType]: the StrEnum compat shim + # makes mypy read a member accessed via `cls` as a plain `str`. + aliases: Dict[str, Any] = { + "missing_variable": cls.missing_var, + "missing_system_var": cls.missing_var, + "missing_tools": cls.missing_tool, + } + if isinstance(value, str): + return aliases.get(value.strip().lower()) + return None + + +class ToolHistoryEntry(BaseModel): + """A tool that must be called, and with which arguments, before deciding.""" + + tool: str + params: Dict[str, str] = Field( + default_factory=dict, + description="Param name -> expression for its value, e.g. 'input.arguments.user_id'", + ) + + +class Requires(BaseModel): + """What a policy item needs in order to be decided.""" + + system_vars: List[str] = Field( + default_factory=list, + description="Subject-variable names the item reads, e.g. ['user_id', 'department']", + ) + tool_history: Optional[List[ToolHistoryEntry]] = Field( + default=None, + description="Tool calls whose results the item needs; None when it needs none", + ) + message_history: Optional[bool] = Field( + default=None, + description="True when the item can only be decided from the conversation", + ) + + +class PendingItem(BaseModel): + """A gap a human must close before the item can be enforced.""" + + type: PendingType + detail: str = Field(..., description="What is missing, and why it blocks the item") + question: str = Field(..., description="The question to put to the user") + suggested_tool: Optional[str] = None + suggested_source: Optional[str] = None + + +class Resolution(BaseModel): + """A human's answer to a pending item or a conflict.""" + + answer: str + decided_by: str + effect: str = Field(..., description="What changed in the spec as a result") + + +class ResolvedItem(PendingItem): + """A :class:`PendingItem` a human has already answered.""" + + resolution: Resolution + + +class PolicyItemV2(BaseModel): + """One enforceable rule, attached to one tool.""" + + id: str = Field(..., description="Stable '.' identifier") + name: str + description: str + compliance_examples: List[str] = Field(default_factory=list) + violation_examples: List[str] = Field(default_factory=list) + references: List[str] = Field( + default_factory=list, description="Verbatim spans of the policy document" + ) + trigger: Trigger = Field(default=Trigger.pre_tool) + requires: Requires = Field(default_factory=Requires) + pending_for_user: List[PendingItem] = Field(default_factory=list) + resolved_by_user: List[ResolvedItem] = Field(default_factory=list) + + +class Conflict(BaseModel): + """Two or more policy items that disagree, and the question that settles it.""" + + id: str + name: str + kind: str = Field(..., description="e.g. 'scope', 'definition', 'dominance'") + conflicting_policies: List[str] = Field( + default_factory=list, description="Policy item ids" + ) + description: str + question: str + resolution: Optional[Resolution] = None + + +class SpecToolInfo(BaseModel): + """What generation learned about the tool itself. + + Named to avoid colliding with ``gen_spec.data_types.ToolInfo`` (a tool + signature, a different concept). Still serializes under ``tool_info``. + """ + + is_read_only: bool = False + user_enrichment: str = "" + + +class SpecDebugV2(BaseModel): + """Generation byproducts. Never read by codegen or the runtime.""" + + tool_info: SpecToolInfo = Field(default_factory=SpecToolInfo) + archive: List[Dict[str, Any]] = Field( + default_factory=list, description="Items dropped by a stage, with the reason" + ) + notes: Optional[List[Any]] = Field( + default=None, + description="Presence-tracked: None means the key is absent on disk", + ) + + +class SpecV2(BaseModel): + """Every policy item attached to one tool, plus its conflicts.""" + + tool_name: str + source_doc: str = "" + policy_items: List[PolicyItemV2] = Field(default_factory=list) + conflicts: List[Conflict] = Field(default_factory=list) + debug: SpecDebugV2 = Field(default_factory=SpecDebugV2) + + def item_ids(self) -> set: + return {item.id for item in self.policy_items} diff --git a/src/toolguard/buildtime/gen_spec_v2/pipeline.py b/src/toolguard/buildtime/gen_spec_v2/pipeline.py new file mode 100644 index 0000000..0de93e8 --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/pipeline.py @@ -0,0 +1,262 @@ +"""Orchestration and the public v2 entry points. + +Three calls, because the caller — not toolguard — knows when a complete spec +set exists: + +- :func:`generate_guard_specs_v2` writes one spec per tool and nothing else. +- :func:`generate_spec_conflicts_v2` reads a complete set and records the + conflicts within it. +- :func:`generate_guard_specs_v2_full` does both, for a plain full build. + +Splitting them keeps partial regeneration safe: regenerating one tool touches +only that tool's file, and the conflicts pass rebuilds conflicts from whatever +is on disk. +""" + +import asyncio +from pathlib import Path +from typing import List, Optional, Sequence + +from loguru import logger +from pydantic import BaseModel, Field + +from toolguard.buildtime.compat.strenum import StrEnum +from toolguard.buildtime.gen_spec.data_types import ToolInfo +from toolguard.buildtime.gen_spec_v2.conflicts import attach_conflicts, find_conflicts +from toolguard.buildtime.gen_spec_v2.context import GenContext +from toolguard.buildtime.gen_spec_v2.models import SpecDebugV2, SpecV2 +from toolguard.buildtime.gen_spec_v2.refmatch import ground_spec +from toolguard.buildtime.gen_spec_v2.serialize import dump_spec, load_specs +from toolguard.buildtime.gen_spec_v2.stages import ( + run_create, + run_enrich, + run_examples, + run_expand, + run_review, +) +from toolguard.buildtime.gen_spec_v2.sysvars import SystemVarsInput +from toolguard.buildtime.gen_spec_v2.tools_input import TOOLS_V2 +from toolguard.buildtime.llm import I_TG_LLM + + +class ToolErrorPolicy(StrEnum): + """What to do when one tool's generation raises.""" + + skip = "skip" + """Record the failure and let every other tool finish.""" + + raise_ = "raise" + """Abort the whole run.""" + + +class SpecV2Options(BaseModel): + """Knobs for a v2 run. The defaults are what a normal build should use.""" + + review_votes: int = Field(default=5, ge=1) + enrich_votes: int = Field(default=3, ge=1) + add_iterations: int = Field(default=3, ge=0) + include_examples: bool = True + example_number: Optional[int] = Field( + default=None, + description="None = let the model choose, >0 = exactly that many per side", + ) + max_concurrency: int = Field(default=8, ge=1) + on_tool_error: str = "skip" + + +async def _generate_one( + llm: I_TG_LLM, + ctx: GenContext, + tool: ToolInfo, + source_doc: str, + options: SpecV2Options, +) -> SpecV2: + """Run create -> expand -> review -> enrich -> examples for one tool.""" + tool_info, items = await run_create(llm, ctx, tool) + items = await run_expand(llm, ctx, tool, items, options.add_iterations) + items, archive = await run_review(llm, ctx, tool, items, options.review_votes) + items = await run_enrich(llm, ctx, tool, items, options.enrich_votes) + + if options.include_examples and options.example_number != 0: + await run_examples(llm, ctx, tool, items, options.example_number) + + spec = SpecV2( + tool_name=tool.name, + source_doc=source_doc, + policy_items=items, + debug=SpecDebugV2(tool_info=tool_info, archive=archive), + ) + ground_spec(spec, ctx.policy_text) + return spec + + +async def generate_guard_specs_v2( + policy_text: str, + tools: TOOLS_V2, + llm: I_TG_LLM, + work_dir: str | Path, + *, + tools2guard: Optional[Sequence[str]] = None, + system_vars: SystemVarsInput = None, + source_doc: str = "", + options: Optional[SpecV2Options] = None, +) -> List[SpecV2]: + """Generate one v2 spec per tool and write it to ``work_dir``. + + Every requested tool gets a file, including one with no applicable rules: + an empty spec is the answer "no rule governs this tool", and its absence + would be indistinguishable from a failed run. + + Writes ``.json``. Note these are the same filenames v1 uses, and v1's + loader will accept them while silently ignoring the v2 fields — point v2 at + its own directory rather than sharing one with v1 output. + """ + options = options or SpecV2Options() + work_dir = Path(work_dir) + work_dir.mkdir(parents=True, exist_ok=True) + + ctx = GenContext.build(policy_text, tools, system_vars) + + if tools2guard is None: + targets = list(ctx.tools) + else: + known = set(ctx.tool_names()) + unknown = [name for name in tools2guard if name not in known] + if unknown: + raise ValueError(f"Unknown tool(s) in tools2guard: {', '.join(unknown)}") + targets = [tool for tool in ctx.tools if tool.name in set(tools2guard)] + + semaphore = asyncio.Semaphore(options.max_concurrency) + + async def guarded(tool: ToolInfo): + async with semaphore: + try: + return await _generate_one(llm, ctx, tool, source_doc, options) + except Exception as ex: # noqa: BLE001 - per-tool isolation by design + if options.on_tool_error == "raise": + raise + logger.error("Spec generation failed for '{}': {}", tool.name, ex) + return None + + results = await asyncio.gather(*[guarded(tool) for tool in targets]) + + specs = [spec for spec in results if spec is not None] + for spec in specs: + dump_spec(spec, work_dir / f"{spec.tool_name}.json") + + logger.debug("gen_spec_v2: wrote {}/{} spec(s)", len(specs), len(targets)) + return specs + + +async def generate_spec_conflicts_v2( + llm: I_TG_LLM, + work_dir: str | Path, + *, + specs: Optional[Sequence[SpecV2]] = None, + rewrite: bool = True, +) -> List[SpecV2]: + """Find conflicts across a complete spec set and record them on each spec. + + Reads ``work_dir`` when ``specs`` is not given, so a caller that generated + tools in separate runs still gets conflicts computed over all of them. Every + spec is rewritten with its conflicts replaced, which makes a rerun + idempotent; ``rewrite=False`` computes without touching disk. + """ + work_dir = Path(work_dir) + spec_list = list(specs) if specs is not None else load_specs(work_dir) + + if not spec_list: + logger.warning("No specs found in {}; nothing to check for conflicts", work_dir) + return [] + + conflicts = await find_conflicts(llm, spec_list) + attach_conflicts(spec_list, conflicts) + + if rewrite: + for spec in spec_list: + if spec.conflicts or (work_dir / f"{spec.tool_name}.json").exists(): + dump_spec(spec, work_dir / f"{spec.tool_name}.json") + + logger.debug( + "gen_spec_v2: {} conflict(s) across {} spec(s)", len(conflicts), len(spec_list) + ) + return spec_list + + +async def generate_guard_examples_v2( + policy_text: str, + tools: TOOLS_V2, + llm: I_TG_LLM, + work_dir: str | Path, + *, + specs: Optional[Sequence[SpecV2]] = None, + system_vars: SystemVarsInput = None, + example_number: Optional[int] = None, + rewrite: bool = True, +) -> List[SpecV2]: + """Rerun only the examples stage for specs already generated. + + The counterpart to v1's ``generate_guard_examples``. Examples are what test + generation works from, so rewording them should not cost a full spec run — + everything else on each item is left exactly as it was. + + Reads ``work_dir`` when ``specs`` is not given. A spec whose tool is absent + from ``tools`` is skipped rather than guessed at: without the tool's + parameters there is nothing concrete to write an example about. + """ + work_dir = Path(work_dir) + spec_list = list(specs) if specs is not None else load_specs(work_dir) + + if not spec_list: + logger.warning( + "No specs found in {}; nothing to regenerate examples for", work_dir + ) + return [] + + ctx = GenContext.build(policy_text, tools, system_vars) + known = set(ctx.tool_names()) + + updated: List[SpecV2] = [] + for spec in spec_list: + if spec.tool_name not in known: + logger.warning( + "Skipping examples for '{}': no such tool in the supplied tool set", + spec.tool_name, + ) + continue + await run_examples( + llm, + ctx, + ctx.tool_by_name(spec.tool_name), + spec.policy_items, + example_number, + ) + if rewrite: + dump_spec(spec, work_dir / f"{spec.tool_name}.json") + updated.append(spec) + + logger.debug("gen_spec_v2: regenerated examples for {} spec(s)", len(updated)) + return updated + + +async def generate_guard_specs_v2_full( + policy_text: str, + tools: TOOLS_V2, + llm: I_TG_LLM, + work_dir: str | Path, + *, + system_vars: SystemVarsInput = None, + source_doc: str = "", + options: Optional[SpecV2Options] = None, +) -> List[SpecV2]: + """Generate every tool's spec, then the conflicts across them.""" + specs = await generate_guard_specs_v2( + policy_text, + tools, + llm, + work_dir, + system_vars=system_vars, + source_doc=source_doc, + options=options, + ) + return await generate_spec_conflicts_v2(llm, work_dir, specs=specs) diff --git a/src/toolguard/buildtime/gen_spec_v2/prompts/__init__.py b/src/toolguard/buildtime/gen_spec_v2/prompts/__init__.py new file mode 100644 index 0000000..b6ef3c5 --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/prompts/__init__.py @@ -0,0 +1,159 @@ +"""System prompts and per-stage user-content assembly. + +The system prompt for each stage is a ``.txt`` file in this package, matching +toolguard's convention; the user content is assembled here, because each stage +needs a different slice of :class:`GenContext` and assembling it in Python +keeps the slice explicit and testable. + +Every stage is shown the system variables and the other tools, whether or not +it obviously needs them: in v1 the feasibility reviewer judged "can this be +validated" without ever being shown the variables that would decide it. +""" + +import json +import os +from functools import lru_cache +from typing import Any, Dict, List + +from toolguard.buildtime.gen_spec.data_types import ToolInfo +from toolguard.buildtime.gen_spec_v2.context import GenContext + +JSON_ONLY_SUFFIX = """ +CRITICAL OUTPUT RULES: +- Output MUST be valid JSON. +- Output MUST match the schema exactly. +- Do NOT include explanations, markdown, comments, or extra text. +- Do NOT wrap the JSON in code fences. +- The first character of the response must be '{' and the last must be '}'. +""" + +STAGES = ("create", "expand", "review", "enrich", "examples", "conflicts") + + +@lru_cache(maxsize=None) +def system(stage: str) -> str: + """The system prompt for ``stage``, with the JSON-only rules appended.""" + if stage not in STAGES: + raise KeyError(f"Unknown stage: {stage}") + path = os.path.join(os.path.dirname(__file__), f"{stage}.txt") + with open(path, "r", encoding="utf-8") as handle: + return handle.read() + JSON_ONLY_SUFFIX + + +def _json(value: Any) -> str: + return json.dumps(value, indent=2, ensure_ascii=False) + + +def _policy_block(ctx: GenContext) -> str: + return f"POLICY DOCUMENT:\n{ctx.render_policy()}" + + +def _system_vars_block(ctx: GenContext) -> str: + return ( + "System variables available (input.extensions.subject.*):\n" + f"{ctx.render_system_vars()}" + ) + + +def _all_tools_block(ctx: GenContext) -> str: + return f"All tools in this agent:\n{ctx.render_tools_overview()}" + + +def _other_tools_block(ctx: GenContext, tool: ToolInfo) -> str: + return f"Other tools available for a prior lookup:\n{ctx.render_other_tools(tool)}" + + +def create_user(ctx: GenContext, tool: ToolInfo) -> str: + return "\n\n".join( + [ + "[STAGE:create]", + "Bind every policy rule that governs or constrains the tool below to it, " + "including cross-cutting rules instantiated for this tool.", + ctx.render_tool_detail(tool), + _policy_block(ctx), + _system_vars_block(ctx), + _all_tools_block(ctx), + "Return only the JSON described in the system prompt.", + ] + ) + + +def expand_user( + ctx: GenContext, tool: ToolInfo, existing_items: List[Dict[str, Any]] +) -> str: + return "\n\n".join( + [ + "[STAGE:expand]", + "Find any additional policy rules applicable to the tool below, beyond " + "the items already captured.", + ctx.render_tool_detail(tool), + _policy_block(ctx), + _system_vars_block(ctx), + _all_tools_block(ctx), + f"Existing policy items for this tool:\n{_json(existing_items)}", + "Return only the JSON described in the system prompt.", + ] + ) + + +def review_user(ctx: GenContext, tool: ToolInfo, item: Dict[str, Any]) -> str: + return "\n\n".join( + [ + "[STAGE:review]", + "Review the policy item below for the given tool.", + ctx.render_tool_detail(tool), + _policy_block(ctx), + _system_vars_block(ctx), + _other_tools_block(ctx, tool), + f"Policy item:\n{_json(item)}", + "Return only the JSON described in the system prompt.", + ] + ) + + +def enrich_user(ctx: GenContext, tool: ToolInfo, item: Dict[str, Any]) -> str: + return "\n\n".join( + [ + "[STAGE:enrich]", + "Enrich the policy item below with trigger, requires, validated " + "references, and any pending_for_user gaps.", + ctx.render_tool_detail(tool), + f"{_policy_block(ctx)}\n\n(Validate the item's references against the " + "document above, verbatim.)", + f"Policy item:\n{_json(item)}", + _system_vars_block(ctx), + _other_tools_block(ctx, tool), + "Return only the JSON described in the system prompt.", + ] + ) + + +def examples_user(ctx: GenContext, tool: ToolInfo, item: Dict[str, Any]) -> str: + return "\n\n".join( + [ + "[STAGE:examples]", + "Write compliance and violation examples for the policy item below, for " + "the given tool.", + ctx.render_tool_detail(tool), + f"Policy item:\n{_json(item)}", + "System variables available (input.extensions.subject.*) — use realistic " + f"values from these when writing examples:\n{ctx.render_system_vars()}", + "Return only the JSON described in the system prompt.", + ] + ) + + +def conflicts_user( + tool_name: str, target: Dict[str, Any], others: List[Dict[str, Any]] +) -> str: + return "\n\n".join( + [ + "[STAGE:conflicts]", + f"Given a TARGET policy item and OTHER policy items from the tool " + f"`{tool_name}`, identify any conflicts between the TARGET and any of " + "the OTHERS.", + f"TARGET:\n{_json(target)}", + f"OTHERS:\n{_json(others)}", + "Return only the JSON described in the system prompt.", + ] + ) diff --git a/src/toolguard/buildtime/gen_spec_v2/prompts/conflicts.txt b/src/toolguard/buildtime/gen_spec_v2/prompts/conflicts.txt new file mode 100644 index 0000000..cbac286 --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/prompts/conflicts.txt @@ -0,0 +1,34 @@ +You look for conflicts between one TARGET policy item and a set of OTHER policy items. + +Given the TARGET and the OTHERS, identify any conflict between the TARGET and any one +of the OTHERS, where a conflict is one of: +- `definition`: the two items define the same term or concept differently. +- `scope`: the two items disagree about which callers, tools, or conditions a rule + applies to. +- `dominance`: the two items would produce contradictory allow/deny outcomes for an + overlapping situation, and it is unclear which should win. + +All of a tool's policy items must hold together (they are ANDed), so a conflict is +worth reporting whenever that conjunction produces an outcome a reader would not expect +— for example a permission granted by one item that another item's prohibition +silently overrides. + +Return ONLY JSON, with exactly this shape: + +{ + "conflicts": [ + { + "id": "conflict..", + "name": str, + "kind": "definition|scope|dominance", + "conflicting_policies": [str, ...], + "description": str, + "question": str + } + ] +} + +`conflicting_policies` MUST list only the `id` values of the given TARGET and OTHER +items that conflict. `description` says when the conflict arises and what each item +would decide; `question` is the question a human must answer to resolve it. Return +`{"conflicts": []}` if none of the OTHERS conflict with the TARGET. diff --git a/src/toolguard/buildtime/gen_spec_v2/prompts/create.txt b/src/toolguard/buildtime/gen_spec_v2/prompts/create.txt new file mode 100644 index 0000000..2e5a822 --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/prompts/create.txt @@ -0,0 +1,68 @@ +You bind natural-language policy rules to ONE tool. + +Given the tool's name, description, and parameters, the full policy document, the +system variables available at policy-evaluation time, and a catalog of every other +tool in this agent, decide which policy rules GOVERN or CONSTRAIN this tool's +invocation or its result. + +A rule governs this tool only if **invoking this tool could actually bring about what +the rule restricts**. Apply this test to every candidate rule: + +> Is there some set of arguments for which calling THIS tool performs the action the +> rule restricts, or produces the state the rule forbids? + +If no possible call of this tool could bring that about, the rule does NOT govern this +tool — even when the rule mentions records, entities, or concepts this tool also touches. +A rule limiting how many passengers a reservation may have governs the tools that set or +change the passenger list; it does not govern a tool that updates baggage or looks up a +reservation, because no arguments to those tools can change the passenger count. A rule +about cancelling something does not govern a tool that reads it or creates it. + +Bind every rule that passes that test, INCLUDING cross-cutting rules instantiated +per-tool: a rule that applies to a whole class of tools (e.g. "confirm with the user +before any write operation" applies to every write tool; "HR may view/edit all +employees' data" applies to every employee-data tool) still governs THIS tool when this +tool is a member of that class — a write tool does perform the write being restricted. +Membership in the class the rule names is what matters, not merely touching the same +data. + +A rule is in scope here when it can be validated using any of: +- this tool's arguments, or the result it returns; +- the available system variables (see that section — these map to + `input.extensions.subject.*`, NOT to tool parameters); +- the result of calling another tool first; +- the conversation history with the user. + +Do not invent rules that are not grounded in the policy document you were given. + +Return ONLY JSON, with exactly this shape: + +{ + "tool_info": {"is_read_only": bool, "user_enrichment": str}, + "policy_items": [ + {"slug": str, "name": str, "description": str, "references": [str, ...]} + ] +} + +Rules for the output: +- Before emitting an item, identify the one thing its rule limits — a specific value or + an action — and check the parameter list for whether some set of arguments to THIS tool + sets, changes, or produces it. **Omit the item when it does not**: a value absent from + this tool's parameters and result cannot be set by this tool, however related the + record and however easily another tool could look it up. +- `slug` is a short, lowercase, underscore-separated summary of the rule, e.g. + `hr_only` or `salary_positive`. Do not include the tool name. +- `name` is a one-line human-readable statement of the rule. +- `description` must be self-contained: a reader who sees only this item, the tool, + and the system variables must be able to decide it without the policy document. It + MUST name (a) which action of THIS tool the rule restricts, and (b) exactly what is + tested — a parameter of this tool, a field of its result, a system variable, or a + value fetched by a prior tool call. If you cannot name both, this tool is not the one + the rule governs: omit the item. +- `references` MUST be a NON-EMPTY list of one or more spans copied EXACTLY — + verbatim, character-for-character, including any markdown — from the POLICY + DOCUMENT below. Each span must be contiguous text from the document and should be + a complete rule, not a fragment. Do NOT paraphrase, summarize, reword, or elide + with "...". +- Do NOT emit a policy item you cannot ground in at least one verbatim span. +- Return no policy items if none of the policy rules apply to this tool. diff --git a/src/toolguard/buildtime/gen_spec_v2/prompts/enrich.txt b/src/toolguard/buildtime/gen_spec_v2/prompts/enrich.txt new file mode 100644 index 0000000..9293edd --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/prompts/enrich.txt @@ -0,0 +1,56 @@ +You enrich a single policy item with the metadata needed to enforce it. + +Given the policy item, the tool it was bound to, the full policy document, the system +variables available, and the other tools in this agent, output: + +`trigger` — when the rule is evaluated: +- `pre_tool`: decidable before the call, from the arguments (plus system variables, a + prior tool lookup, or the conversation). +- `post_tool`: only decidable against what the tool RETURNS, or it describes something + that must happen AFTER the call succeeds. + +`requires` — what else is needed to evaluate it: +- `system_vars`: the names of the system variables it reads. Use ONLY names from the + system variables section below; never invent one. +- `tool_history`: the prior tool calls whose results it depends on, as a list of + `{"tool": str, "params": {}}`, or `null` when it needs none. Use ONLY tool names + listed below. Each `params` value must be an expression over `input.arguments.*` (this + tool's arguments) or `input.extensions.subject.*` (system variables). +- `message_history`: `true` when it can only be decided from the conversation with the + user (for example, a rule requiring the user to have confirmed something, or to have + supplied a detail in chat), otherwise `null`. + +`references` — the item already has references, shown in the item JSON below. VALIDATE +them; do not choose or generate new ones. Return the subset that appear EXACTLY, +character-for-character, in the POLICY DOCUMENT below. Drop any that do not. Return +the list unchanged if all of them are already verbatim. Return `[]` if none validate. + +`pending_for_user` — anything the rule needs that is NOT available from this tool's +arguments or result, the listed system variables, or the listed tools. One entry each: +- `{"type": "missing_tool", "detail": str, "question": str, "suggested_tool": str}` — + the rule needs a capability no listed tool provides. +- `{"type": "missing_var", "detail": str, "question": str, "suggested_source": str}` — + the rule needs an attribute the system variables do not supply. +- `{"type": "clarification", "detail": str, "question": str}` — the rule is ambiguous + and a human must choose the deterministic reading before it can be enforced. + +Use exactly those three type strings. `detail` says what is missing and why it blocks +the rule; `question` is the question to put to a human. Return `[]` when nothing is +missing. Needing a system variable, a prior tool call, or the conversation history is +NOT a gap — declare it in `requires` instead. + +Return ONLY JSON, with exactly this shape: + +{ + "trigger": "pre_tool|post_tool", + "requires": { + "system_vars": [str, ...], + "tool_history": [{"tool": str, "params": {}}] | null, + "message_history": true | null + }, + "references": [str, ...], + "pending_for_user": [ + {"type": "missing_tool|missing_var|clarification", "detail": str, + "question": str, "suggested_tool": str, "suggested_source": str} + ] +} diff --git a/src/toolguard/buildtime/gen_spec_v2/prompts/examples.txt b/src/toolguard/buildtime/gen_spec_v2/prompts/examples.txt new file mode 100644 index 0000000..d9746a6 --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/prompts/examples.txt @@ -0,0 +1,22 @@ +You write short natural-language example scenarios for a single policy item, +illustrating how it should behave when the given tool is called. + +Given the policy item, the tool, and the system variables available (with their +realistic allowed values), write: +- `compliance_examples`: short scenarios where the tool call should be ALLOWED because + it satisfies the rule. Include a scenario where the rule simply does not apply (an + optional argument the rule tests is absent), when that is possible. +- `violation_examples`: short scenarios where the tool call should be DENIED because it + violates the rule. Cover the boundary case where the rule is strict (e.g. exactly at + a limit that must be exceeded). + +Each example is a short, concrete, natural-language sentence describing a plausible +call and its context — who is calling, with what arguments, under what circumstances — +not code, not JSON. Use realistic values drawn from the system variables shown below +(an actual department or organization name) rather than placeholders. When the item's +trigger is `post_tool`, describe the result the tool returned rather than only the +arguments. + +Return ONLY JSON, with exactly this shape: + +{"compliance_examples": [str, ...], "violation_examples": [str, ...]} diff --git a/src/toolguard/buildtime/gen_spec_v2/prompts/expand.txt b/src/toolguard/buildtime/gen_spec_v2/prompts/expand.txt new file mode 100644 index 0000000..1f4c390 --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/prompts/expand.txt @@ -0,0 +1,43 @@ +You expand the set of policy items already captured for ONE tool. + +Given the tool, the full policy document, the system variables available, a catalog of +every other tool in this agent, and the policy items already captured for this tool, +list any ADDITIONAL applicable rules that were missed. Do not repeat items that are +already captured (matched by name or by substantially the same rule). + +Apply the same test as before to every candidate: a rule governs this tool only if there +is some set of arguments for which calling THIS tool performs the action the rule +restricts, or produces the state the rule forbids. If no possible call of this tool could +bring that about, do not propose it — sharing a record or a concept with the rule is not +enough. + +Missed rules commonly include: +- cross-cutting rules instantiated per-tool: a rule that applies to a whole class of + tools (e.g. "confirm with the user before any write operation" applies to every + write tool; "HR may view/edit all employees' data" applies to every employee-data + tool) still governs THIS tool when this tool is a member of that class; +- rules about what the tool's RESULT may contain or return, not only about whether it + may be called; +- data-integrity rules on individual arguments (value ranges, formats, date ordering); +- rules that need a prior tool call or the conversation history to decide — those are + in scope, not out of it. + +Return ONLY JSON, with exactly this shape: + +{ + "policy_items": [ + {"slug": str, "name": str, "description": str, "references": [str, ...]} + ] +} + +Rules for the output: +- `slug` is a short, lowercase, underscore-separated summary of the rule. Do not + include the tool name. +- `description` must be self-contained, naming the exact argument, system variable, or + result field it tests. +- `references` MUST be a NON-EMPTY list of spans copied EXACTLY — verbatim, + character-for-character, including any markdown — from the POLICY DOCUMENT below. + Each span must be contiguous text and should be a complete rule. Do NOT paraphrase + or elide with "...". +- Do NOT propose a policy item you cannot ground in at least one verbatim span. +- Return `{"policy_items": []}` if there are no additional applicable rules. diff --git a/src/toolguard/buildtime/gen_spec_v2/prompts/review.txt b/src/toolguard/buildtime/gen_spec_v2/prompts/review.txt new file mode 100644 index 0000000..7252517 --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/prompts/review.txt @@ -0,0 +1,51 @@ +You judge a single policy item against a single tool, on exactly two criteria. + +`is_relevant`: does this rule GOVERN or CONSTRAIN this tool's invocation or its result? + +Decide it with this test: + +> Is there some set of arguments for which calling THIS tool performs the action the +> rule restricts, or produces the state the rule forbids? + +Answer **false** when no possible call of this tool could bring that about. Sharing a +record, an entity, or a concept with the rule is not enough: a rule capping a +reservation's passenger count is not relevant to a tool that updates baggage or reads a +reservation, because no arguments to those tools can change the passenger count. A rule +about cancelling is not relevant to a tool that reads or creates. A rule the item's own +description cannot tie to a specific action of THIS tool is not relevant to it. + +Answer **true** for a genuine cross-cutting rule — one that also applies to other tools +(e.g. "confirm with the user before any write operation", which applies to every write +tool; or "HR may view/edit all employees' data", which applies to every employee-data +tool) — when THIS tool is a member of the class the rule names, because then calling it +does perform the restricted action. Being shared across tools is not itself a reason to +reject. + +`can_be_validated`: can it be checked using any of +- this tool's arguments; +- the result this tool returns (a rule evaluated after the call is still valid); +- the AVAILABLE SYSTEM VARIABLES (`input.extensions.subject.*` — e.g. department, + organization, user_id, shown in the system variables section below); +- a prior lookup with one of the other tools listed below; +- the conversation history with the user? + +Do NOT require the data to be a parameter of this tool — department, organization, +user_id and similar attributes of the acting user come from system variables, not from +tool arguments. Do NOT answer false merely because enforcement needs something beyond +the arguments; answer false only when none of the five sources above could decide it, +or when deciding it would require a subjective judgment no deterministic check can make. + +Before answering, identify the one thing this rule limits — a specific value (quote the +tool parameter or result field by name) or an action — and whether some set of arguments +to THIS tool sets, changes, or produces it. Read the tool's parameter list. A value that +is not among this tool's parameters and not part of its result cannot be set by this +tool, however related the record and however easily another tool could look it up; a rule +limiting such a value is not relevant here. State that finding in `reason`. + +Return ONLY JSON, with exactly this shape: + +{ + "is_relevant": bool, + "can_be_validated": bool, + "reason": str +} diff --git a/src/toolguard/buildtime/gen_spec_v2/reconcile.py b/src/toolguard/buildtime/gen_spec_v2/reconcile.py new file mode 100644 index 0000000..166be82 --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/reconcile.py @@ -0,0 +1,153 @@ +"""Combining repeated enrich votes into one answer. + +The enrich stage asks the model the same question several times, because a +single answer about "does this rule need the chat history?" is unreliable. +This module reduces those answers to one, with deliberately asymmetric rules: + +- **system_vars** unions. A missed variable produces a guard that reads the + wrong thing; a spurious one only over-declares. +- **message_history** and a null **tool_history** need a strict majority. Both + make an item unenforceable by today's codegen, so one stray vote must not + decide it. +- **trigger** ties resolve to ``pre_tool``: a pre-tool guard that should have + been post-tool blocks a call, while the reverse lets one through. + +Votes come from an LLM, so every field is read defensively — a malformed vote +is skipped, never fatal. +""" + +from typing import Any, Dict, List, Optional, Sequence + +from loguru import logger +from pydantic import BaseModel, Field, ValidationError + +from toolguard.buildtime.gen_spec_v2.models import ( + PendingItem, + Requires, + ToolHistoryEntry, + Trigger, +) + + +class Reconciled(BaseModel): + """One answer, reduced from many votes.""" + + trigger: Trigger = Trigger.pre_tool + requires: Requires = Field(default_factory=Requires) + references: List[str] = Field(default_factory=list) + pending_for_user: List[PendingItem] = Field(default_factory=list) + + +def _requires_of(vote: Dict[str, Any]) -> Dict[str, Any]: + requires = vote.get("requires") + return requires if isinstance(requires, dict) else {} + + +def _str_list(value: Any) -> List[str]: + if not isinstance(value, list): + return [] + return [v for v in value if isinstance(v, str)] + + +def _tool_history_of(vote: Dict[str, Any]) -> List[Dict[str, Any]]: + raw = _requires_of(vote).get("tool_history") + if not isinstance(raw, list): + return [] + return [entry for entry in raw if isinstance(entry, dict) and entry.get("tool")] + + +def _reconcile_trigger(votes: Sequence[Dict[str, Any]]) -> Trigger: + post = sum(1 for v in votes if v.get("trigger") == Trigger.post_tool.value) + pre = sum(1 for v in votes if v.get("trigger") == Trigger.pre_tool.value) + return Trigger.post_tool if post > pre else Trigger.pre_tool + + +def _reconcile_tool_history( + votes: Sequence[Dict[str, Any]], +) -> Optional[List[ToolHistoryEntry]]: + histories = [_tool_history_of(v) for v in votes] + empty = sum(1 for h in histories if not h) + if votes and empty * 2 > len(votes): + return None + + candidates = [h for h in histories if h] + if not candidates: + return None + + longest = max(candidates, key=len) + entries = [] + for entry in longest: + params = entry.get("params") + entries.append( + ToolHistoryEntry( + tool=str(entry["tool"]), + params={ + str(k): str(v) + for k, v in (params.items() if isinstance(params, dict) else []) + }, + ) + ) + return entries or None + + +def _reconcile_message_history(votes: Sequence[Dict[str, Any]]) -> Optional[bool]: + """``True`` on a strict majority, otherwise ``None`` — the format has no false.""" + wanted = sum(1 for v in votes if _requires_of(v).get("message_history") is True) + return True if votes and wanted * 2 > len(votes) else None + + +def _reconcile_references(votes: Sequence[Dict[str, Any]]) -> List[str]: + references: List[str] = [] + for vote in votes: + for reference in _str_list(vote.get("references")): + if reference not in references: + references.append(reference) + return references + + +def _reconcile_pending(votes: Sequence[Dict[str, Any]]) -> List[PendingItem]: + """Dedupe by ``(type, question)``, first occurrence winning. + + An entry that will not validate — unknown type, no question — is dropped: + a gap nobody can act on is worse than no gap at all. + """ + pending: List[PendingItem] = [] + seen = set() + for vote in votes: + raw_entries = vote.get("pending_for_user") + if not isinstance(raw_entries, list): + continue + for raw in raw_entries: + if not isinstance(raw, dict): + continue + try: + entry = PendingItem.model_validate(raw) + except ValidationError as ex: + logger.warning("Dropping unusable pending entry {}: {}", raw, ex.title) + continue + key = (entry.type, entry.question) + if key in seen: + continue + seen.add(key) + pending.append(entry) + return pending + + +def reconcile(votes: Sequence[Any]) -> Reconciled: + """Reduce ``votes`` to one answer. Non-dict votes are ignored.""" + usable = [v for v in votes if isinstance(v, dict)] + + system_vars = sorted( + {sv for v in usable for sv in _str_list(_requires_of(v).get("system_vars"))} + ) + + return Reconciled( + trigger=_reconcile_trigger(usable), + requires=Requires( + system_vars=system_vars, + tool_history=_reconcile_tool_history(usable), + message_history=_reconcile_message_history(usable), + ), + references=_reconcile_references(usable), + pending_for_user=_reconcile_pending(usable), + ) diff --git a/src/toolguard/buildtime/gen_spec_v2/refmatch.py b/src/toolguard/buildtime/gen_spec_v2/refmatch.py new file mode 100644 index 0000000..911c4b8 --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/refmatch.py @@ -0,0 +1,302 @@ +"""Grounding quoted references back onto the policy document. + +A reference is the evidence for a policy item: the span of the document the +item came from. The LLM quotes it, and quotes drift — line wrapping, dropped +markdown emphasis, a hyphen for an em-dash, a fragment instead of the whole +rule. This module maps a drifted quote back onto the exact original text, or +reports that it cannot. + +Design notes, each replacing a defect in v1's ``find_mismatched_references``: + +- :func:`normalize` returns an offset map alongside the normalized text, so a + match can always be projected back to an exact original substring. v1 added + the *reference's* length to a normalized index, which is only correct while + normalization preserves length. +- Matching is against document *segments* (bullets, sentences), so a partial + match snaps out to the whole rule instead of returning a fragment. +- The fallback requires **consecutive** segments, where v1 accepted any two + fragments found anywhere in the document. +- A quote that cannot be grounded is reported, not silently kept. +""" + +import re +import unicodedata +from difflib import SequenceMatcher +from typing import Dict, List, Tuple + +from loguru import logger +from pydantic import BaseModel + +from toolguard.buildtime.gen_spec_v2.models import SpecV2 + +FUZZY_THRESHOLD = 0.75 +"""Minimum SequenceMatcher ratio for a quote to be considered the same rule.""" + +SNAP_COVERAGE = 0.6 +"""Fraction of a segment a match must cover before it snaps out to the whole segment.""" + +MAX_SPAN_SEGMENTS = 4 +"""Longest run of consecutive segments a single quote may be grounded to.""" + +_DASHES = "‐‑‒–—―−" +_SINGLE_QUOTES = "‘’‚‛′" +_DOUBLE_QUOTES = "“”„‟″" +_DROPPED = "*_`~" +"""Markdown emphasis and code markers, dropped on both sides of a comparison.""" + +_BULLET = re.compile(r"^(\s*[-*+]\s+)(.*\S)\s*$") +_HEADING = re.compile(r"^\s*#{1,6}\s") +_SENTENCE_END = re.compile(r"(?<=[.!?])\s+") + + +class Span(BaseModel): + """A stretch of the original document, with its exact text.""" + + start: int + end: int + text: str + + +def normalize(text: str) -> Tuple[str, List[int]]: + """Normalize ``text`` for comparison and map each output char to its source. + + Casefolds, unifies dashes and quotes, drops markdown emphasis, and + collapses whitespace runs to a single space. ``offsets[i]`` is the index in + ``text`` that produced ``norm[i]``, so any match on the normalized string + projects back onto an exact original substring. + """ + out: List[str] = [] + offsets: List[int] = [] + pending_space = False + + for index, char in enumerate(text): + if char.isspace(): + pending_space = bool(out) + continue + + if char in _DROPPED: + continue + + if pending_space: + out.append(" ") + offsets.append(index) + pending_space = False + + if char in _DASHES: + replacement = "-" + elif char in _SINGLE_QUOTES: + replacement = "'" + elif char in _DOUBLE_QUOTES: + replacement = '"' + else: + replacement = unicodedata.normalize("NFKC", char).lower() + + for out_char in replacement: + out.append(out_char) + offsets.append(index) + + return "".join(out), offsets + + +def _norm(text: str) -> str: + return normalize(text)[0] + + +def segments(doc: str) -> List[Span]: + """Split ``doc`` into candidate reference units, with original spans. + + A markdown bullet is one unit with its marker excluded, since that is how + a rule reads as a quote. Other prose is split into sentences. Headings and + blank lines yield no units: nothing quotes them as a rule. + """ + spans: List[Span] = [] + offset = 0 + + for line in doc.splitlines(keepends=True): + stripped = line.strip() + if not stripped or _HEADING.match(line): + offset += len(line) + continue + + bullet = _BULLET.match(line.rstrip("\n")) + if bullet: + start = offset + len(bullet.group(1)) + text = bullet.group(2) + spans.append(Span(start=start, end=start + len(text), text=text)) + offset += len(line) + continue + + # Prose: one span per sentence, positioned within the line. + line_body = line.rstrip("\n") + indent = len(line_body) - len(line_body.lstrip()) + cursor = offset + indent + for sentence in _SENTENCE_END.split(line_body.strip()): + if not sentence: + continue + start = doc.find(sentence, cursor) + if start < 0: + continue + spans.append(Span(start=start, end=start + len(sentence), text=sentence)) + cursor = start + len(sentence) + offset += len(line) + + return spans + + +def _overlapping(spans: List[Span], start: int, end: int) -> List[Span]: + return [s for s in spans if s.start < end and start < s.end] + + +def _exact(reference: str, doc: str, spans: List[Span]) -> List[str]: + """Exact normalized substring match, snapped out to whole segments.""" + norm_doc, offsets = normalize(doc) + norm_ref = _norm(reference) + if not norm_ref: + return [] + + index = norm_doc.find(norm_ref) + if index < 0: + return [] + + start = offsets[index] + end = offsets[index + len(norm_ref) - 1] + 1 + touched = _overlapping(spans, start, end) + + if len(touched) > 1: + return [s.text for s in touched] + + if touched: + segment = touched[0] + covered = min(end, segment.end) - max(start, segment.start) + if covered >= SNAP_COVERAGE * (segment.end - segment.start): + return [segment.text] + + return [doc[start:end]] + + +def _best_segment(norm_ref: str, spans: List[Span]) -> List[str]: + """The single segment most similar to the quote, if similar enough.""" + best_ratio = 0.0 + best: List[str] = [] + for segment in spans: + ratio = SequenceMatcher(None, norm_ref, _norm(segment.text)).ratio() + if ratio > best_ratio: + best_ratio, best = ratio, [segment.text] + return best if best_ratio >= FUZZY_THRESHOLD else [] + + +def _best_run(norm_ref: str, spans: List[Span]) -> List[str]: + """The best run of consecutive segments, for a quote spanning a boundary.""" + best_ratio = 0.0 + best: List[str] = [] + for size in range(2, MAX_SPAN_SEGMENTS + 1): + for i in range(len(spans) - size + 1): + run = spans[i : i + size] + joined = _norm(" ".join(s.text for s in run)) + ratio = SequenceMatcher(None, norm_ref, joined).ratio() + if ratio > best_ratio: + best_ratio, best = ratio, [s.text for s in run] + return best if best_ratio >= FUZZY_THRESHOLD else [] + + +def ground(reference: str, doc: str) -> List[str]: + """Return the document spans ``reference`` quotes, or ``[]`` if none. + + Tries an exact normalized match first, then the closest single segment, + then the closest run of consecutive segments. More than one span comes + back when the quote genuinely crosses a segment boundary. + """ + spans = segments(doc) + if not spans: + return [] + + exact = _exact(reference, doc, spans) + if exact: + return exact + + norm_ref = _norm(reference) + if not norm_ref: + return [] + + return _best_segment(norm_ref, spans) or _best_run(norm_ref, spans) + + +def ground_spec(spec: SpecV2, policy_text: str) -> List[str]: + """Rewrite every item's references to verbatim spans of ``policy_text``. + + Grounded quotes are replaced (deduped, first-seen order). An item that keeps + at least one grounded quote survives with its ungrounded ones intact — + dropping those would weaken an otherwise sound item — but they are logged and + recorded in ``spec.debug.notes`` so they are never mistaken for real + citations. + + An item where **nothing** grounded is archived instead of kept. Every quote + being absent from the policy document means the item is not a rule from this + policy: in practice it has quoted a tool's own description. Keeping it would + put a rule nobody wrote into the spec. + + Returns every ungrounded quote, from surviving and archived items alike. + """ + ungrounded: List[str] = [] + per_item: Dict[str, List[str]] = {} + kept = [] + + for item in spec.policy_items: + resolved: List[str] = [] + item_ungrounded: List[str] = [] + grounded_any = False + + for reference in item.references: + spans = ground(reference, policy_text) + if not spans: + item_ungrounded.append(reference) + if reference not in resolved: + resolved.append(reference) + continue + grounded_any = True + for text in spans: + if text not in resolved: + resolved.append(text) + + item.references = resolved + if item_ungrounded: + ungrounded.extend(item_ungrounded) + + if item.references and not grounded_any: + logger.warning( + "{}: archived — none of its {} reference(s) appear in the policy " + "document", + item.id, + len(item.references), + ) + spec.debug.archive.append( + { + "id": item.id, + "name": item.name, + "reason": ( + "No reference could be located in the policy document, so the " + "item is not grounded in this policy." + ), + "stage": "refmatch", + "references": list(item.references), + } + ) + continue + + if item_ungrounded: + per_item[item.id] = item_ungrounded + logger.warning( + "{}: {} reference(s) could not be grounded in the policy document", + item.id, + len(item_ungrounded), + ) + kept.append(item) + + spec.policy_items = kept + + if per_item: + notes = list(spec.debug.notes or []) + notes.append({"stage": "refmatch", "ungrounded_references": per_item}) + spec.debug.notes = notes + + return ungrounded diff --git a/src/toolguard/buildtime/gen_spec_v2/serialize.py b/src/toolguard/buildtime/gen_spec_v2/serialize.py new file mode 100644 index 0000000..8d4d674 --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/serialize.py @@ -0,0 +1,159 @@ +"""Reading and writing v2 spec files in the format's fixed key order. + +Pydantic serializes in declaration order and always emits every field, but +the on-disk format fixes both the key order and a set of omit-when-empty +keys. This module owns that mapping in both directions so +:mod:`toolguard.buildtime.gen_spec_v2.models` stays a plain schema. + +Output is byte-identical to the ground-truth specs under +``tests/data/specs_v2/``, which is what ``test_serialize.py`` asserts. +""" + +import json +from pathlib import Path +from typing import Any, Dict, List, Optional + +from toolguard.buildtime.gen_spec_v2.models import ( + Conflict, + PendingItem, + PolicyItemV2, + Requires, + ResolvedItem, + Resolution, + SpecDebugV2, + SpecV2, +) + + +def _requires_to_dict(requires: Requires) -> Dict[str, Any]: + """All three keys are always present, even when empty or null.""" + return { + "system_vars": list(requires.system_vars), + "tool_history": ( + None + if requires.tool_history is None + else [ + {"tool": entry.tool, "params": dict(entry.params)} + for entry in requires.tool_history + ] + ), + "message_history": requires.message_history, + } + + +def _resolution_to_dict(resolution: Optional[Resolution]) -> Optional[Dict[str, Any]]: + if resolution is None: + return None + return { + "answer": resolution.answer, + "decided_by": resolution.decided_by, + "effect": resolution.effect, + } + + +def _pending_to_dict(pending: PendingItem) -> Dict[str, Any]: + """``question`` follows the optional suggestions, matching the format.""" + d: Dict[str, Any] = {"type": pending.type.value, "detail": pending.detail} + if pending.suggested_tool is not None: + d["suggested_tool"] = pending.suggested_tool + if pending.suggested_source is not None: + d["suggested_source"] = pending.suggested_source + d["question"] = pending.question + if isinstance(pending, ResolvedItem): + d["resolution"] = _resolution_to_dict(pending.resolution) + return d + + +def _item_to_dict(item: PolicyItemV2) -> Dict[str, Any]: + d: Dict[str, Any] = { + "id": item.id, + "name": item.name, + "description": item.description, + "compliance_examples": list(item.compliance_examples), + "violation_examples": list(item.violation_examples), + "references": list(item.references), + "trigger": item.trigger.value, + "requires": _requires_to_dict(item.requires), + } + if item.pending_for_user: + d["pending_for_user"] = [_pending_to_dict(p) for p in item.pending_for_user] + if item.resolved_by_user: + d["resolved_by_user"] = [_pending_to_dict(r) for r in item.resolved_by_user] + return d + + +def _conflict_to_dict(conflict: Conflict) -> Dict[str, Any]: + return { + "id": conflict.id, + "name": conflict.name, + "kind": conflict.kind, + "conflicting_policies": list(conflict.conflicting_policies), + "description": conflict.description, + "question": conflict.question, + "resolution": _resolution_to_dict(conflict.resolution), + } + + +def _debug_to_dict(debug: SpecDebugV2) -> Dict[str, Any]: + d: Dict[str, Any] = { + "tool_info": { + "is_read_only": debug.tool_info.is_read_only, + "user_enrichment": debug.tool_info.user_enrichment, + }, + "archive": list(debug.archive), + } + if debug.notes is not None: + d["notes"] = list(debug.notes) + return d + + +def spec_to_dict(spec: SpecV2) -> Dict[str, Any]: + """Serialize ``spec`` in the fixed on-disk key order. + + Omitted when empty: ``conflicts`` on the spec, ``pending_for_user`` and + ``resolved_by_user`` on an item, ``suggested_tool`` / ``suggested_source`` + on a pending item, and ``debug.notes``. Everything else is always + present, even when empty. + """ + d: Dict[str, Any] = { + "tool_name": spec.tool_name, + "source_doc": spec.source_doc, + "policy_items": [_item_to_dict(item) for item in spec.policy_items], + } + if spec.conflicts: + d["conflicts"] = [_conflict_to_dict(c) for c in spec.conflicts] + d["debug"] = _debug_to_dict(spec.debug) + return d + + +def spec_from_dict(d: Dict[str, Any]) -> SpecV2: + """Inverse of :func:`spec_to_dict`; missing optional keys default to empty.""" + return SpecV2.model_validate(d) + + +def dump_spec_str(spec: SpecV2) -> str: + """Render ``spec`` as the exact text written to disk, trailing newline included.""" + return json.dumps(spec_to_dict(spec), indent=2, ensure_ascii=False) + "\n" + + +def dump_spec(spec: SpecV2, path: str | Path) -> None: + Path(path).write_text(dump_spec_str(spec), encoding="utf-8") + + +def load_spec(path: str | Path) -> SpecV2: + return spec_from_dict(json.loads(Path(path).read_text(encoding="utf-8"))) + + +def load_specs(directory: str | Path, skip: Optional[List[str]] = None) -> List[SpecV2]: + """Load every ``.json`` in ``directory``, sorted by filename. + + Files whose stem is in ``skip`` are ignored, as is anything starting with + ``_`` (e.g. a caller's ``_manifest.json``). + """ + skipped = set(skip or ()) + specs = [] + for path in sorted(Path(directory).glob("*.json")): + if path.name.startswith("_") or path.stem in skipped: + continue + specs.append(load_spec(path)) + return specs diff --git a/src/toolguard/buildtime/gen_spec_v2/stages/__init__.py b/src/toolguard/buildtime/gen_spec_v2/stages/__init__.py new file mode 100644 index 0000000..5769e5d --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/stages/__init__.py @@ -0,0 +1,9 @@ +"""The per-tool generation stages, in pipeline order.""" + +from toolguard.buildtime.gen_spec_v2.stages.create import run_create +from toolguard.buildtime.gen_spec_v2.stages.enrich import run_enrich +from toolguard.buildtime.gen_spec_v2.stages.examples import run_examples +from toolguard.buildtime.gen_spec_v2.stages.expand import run_expand +from toolguard.buildtime.gen_spec_v2.stages.review import run_review + +__all__ = ["run_create", "run_expand", "run_review", "run_enrich", "run_examples"] diff --git a/src/toolguard/buildtime/gen_spec_v2/stages/_shared.py b/src/toolguard/buildtime/gen_spec_v2/stages/_shared.py new file mode 100644 index 0000000..da07209 --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/stages/_shared.py @@ -0,0 +1,87 @@ +"""Helpers every stage needs for talking to the LLM and reading its output. + +LLM responses are the one input no stage controls, so every read goes through +a defensive accessor here: a response that is valid JSON but the wrong shape +degrades to a default rather than aborting a whole tool's generation. +""" + +from typing import Any, Dict, List, Optional + +from toolguard.buildtime.gen_spec_v2.models import ( + PolicyItemV2, + slugify, + unique_id, +) + + +def messages(system: str, user: str) -> List[Dict[str, str]]: + return [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + + +def dict_list(response: Any, key: str) -> List[Dict[str, Any]]: + """The list of dicts at ``response[key]``, or ``[]`` for any other shape.""" + if not isinstance(response, dict): + return [] + raw = response.get(key) + if not isinstance(raw, list): + return [] + return [entry for entry in raw if isinstance(entry, dict)] + + +def str_list(value: Any) -> List[str]: + if not isinstance(value, list): + return [] + return [v for v in value if isinstance(v, str) and v.strip()] + + +def item_summary(item: PolicyItemV2) -> Dict[str, Any]: + """The item as the prompts show it — id, name, description, references.""" + return { + "id": item.id, + "name": item.name, + "description": item.description, + "references": list(item.references), + } + + +def build_item( + raw: Dict[str, Any], tool_name: str, taken: set +) -> Optional[PolicyItemV2]: + """Build one item from a raw create/expand entry, or ``None`` to drop it. + + Dropped when unnamed, or when it quotes nothing: an item with no reference + is not grounded in the policy document, and there is no way to check later + whether it was invented. + + The id is always assigned here, never taken from the model: it must be + unique within the tool and well-formed, since it is the handle conflicts + and pending gaps refer to. + """ + name = raw.get("name") + if not isinstance(name, str) or not name.strip(): + return None + + references = str_list(raw.get("references")) + if not references: + return None + + slug = raw.get("slug") or "" + if not isinstance(slug, str) or not slug.strip(): + # Some models answer with a fully-qualified id instead of a slug. + raw_id = raw.get("id") + slug = raw_id.split(".", 1)[-1] if isinstance(raw_id, str) else "" + slug = slugify(slug) or slugify(name) + + item_id = unique_id(tool_name, slug, taken) + taken.add(item_id) + + description = raw.get("description") + return PolicyItemV2( + id=item_id, + name=name.strip(), + description=description if isinstance(description, str) else "", + references=references, + ) diff --git a/src/toolguard/buildtime/gen_spec_v2/stages/create.py b/src/toolguard/buildtime/gen_spec_v2/stages/create.py new file mode 100644 index 0000000..2237053 --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/stages/create.py @@ -0,0 +1,50 @@ +"""The ``create`` stage: bind the policy document's rules to one tool.""" + +from typing import List, Tuple + +from loguru import logger + +from toolguard.buildtime.gen_spec.data_types import ToolInfo +from toolguard.buildtime.gen_spec_v2 import prompts +from toolguard.buildtime.gen_spec_v2.context import GenContext +from toolguard.buildtime.gen_spec_v2.models import PolicyItemV2, SpecToolInfo +from toolguard.buildtime.gen_spec_v2.stages._shared import ( + build_item, + dict_list, + messages, +) +from toolguard.buildtime.llm import I_TG_LLM + + +async def run_create( + llm: I_TG_LLM, ctx: GenContext, tool: ToolInfo +) -> Tuple[SpecToolInfo, List[PolicyItemV2]]: + """Bind every applicable rule to ``tool`` in one call. + + Returns what the model learned about the tool itself, plus one item per + grounded rule with placeholder trigger/requires/examples for the later + stages to fill in. + """ + response = await llm.chat_json( + messages(prompts.system("create"), prompts.create_user(ctx, tool)) + ) + + raw_info = response.get("tool_info") if isinstance(response, dict) else None + tool_info = SpecToolInfo() + if isinstance(raw_info, dict): + tool_info = SpecToolInfo( + is_read_only=bool(raw_info.get("is_read_only", False)), + user_enrichment=str(raw_info.get("user_enrichment", "") or ""), + ) + + taken: set = set() + items: List[PolicyItemV2] = [] + for raw in dict_list(response, "policy_items"): + item = build_item(raw, tool.name, taken) + if item is None: + logger.debug("{}: dropping ungrounded or unnamed create item", tool.name) + continue + items.append(item) + + logger.debug("create({}): {} item(s)", tool.name, len(items)) + return tool_info, items diff --git a/src/toolguard/buildtime/gen_spec_v2/stages/enrich.py b/src/toolguard/buildtime/gen_spec_v2/stages/enrich.py new file mode 100644 index 0000000..fc224c9 --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/stages/enrich.py @@ -0,0 +1,111 @@ +"""The ``enrich`` stage: when a rule applies, what it needs, and what's missing. + +This is where a v2 spec earns its shape. Each item gets several votes on one +question — how would you actually enforce this? — reconciled by a pure +function, then filtered against reality: a system variable nobody declared or +a tool that does not exist is dropped rather than written into the spec. + +An item whose gap cannot be closed is **kept**, carrying its +``pending_for_user`` alert. Surfacing the gap is the point of the field; the +adapter is what stops such an item reaching codegen. +""" + +import asyncio +from typing import List + +from loguru import logger + +from toolguard.buildtime.gen_spec.data_types import ToolInfo +from toolguard.buildtime.gen_spec_v2 import prompts +from toolguard.buildtime.gen_spec_v2.context import GenContext +from toolguard.buildtime.gen_spec_v2.models import PolicyItemV2, Requires +from toolguard.buildtime.gen_spec_v2.reconcile import Reconciled, reconcile +from toolguard.buildtime.gen_spec_v2.stages._shared import item_summary, messages +from toolguard.buildtime.gen_spec_v2.sysvars import keep_declared +from toolguard.buildtime.llm import I_TG_LLM + + +def _validated_requires( + reconciled: Reconciled, ctx: GenContext, item_id: str +) -> Requires: + """Drop anything the runtime could not actually supply. + + An undeclared system variable or an unknown tool name would compile into a + guard reading something that does not exist, so neither is trusted just + because the model named it. + """ + known_tools = set(ctx.tool_names()) + history = reconciled.requires.tool_history or [] + usable = [] + for entry in history: + if entry.tool in known_tools: + usable.append(entry) + else: + logger.warning( + "{}: dropping tool_history entry for unknown tool '{}'", + item_id, + entry.tool, + ) + + return Requires( + system_vars=keep_declared( + reconciled.requires.system_vars, ctx.system_vars, context=item_id + ), + tool_history=usable or None, + message_history=reconciled.requires.message_history, + ) + + +def _validated_references(reconciled: Reconciled, item: PolicyItemV2) -> List[str]: + """Keep or drop the item's references — never add. + + The votes only validate; intersecting with what the item arrived with means + a stray vote cannot introduce a reference that was never in the document. + Validation that removes everything falls back to the original set: an item + with no evidence at all is worse than one with imperfect evidence. + """ + original = list(item.references) + validated = [r for r in reconciled.references if r in original] + return validated or original + + +async def _enrich_item( + llm: I_TG_LLM, ctx: GenContext, tool: ToolInfo, item: PolicyItemV2, votes: int +) -> Reconciled: + system = prompts.system("enrich") + user = prompts.enrich_user(ctx, tool, item_summary(item)) + results = await asyncio.gather( + *[llm.chat_json(messages(system, user)) for _ in range(votes)] + ) + return reconcile(list(results)) + + +async def run_enrich( + llm: I_TG_LLM, + ctx: GenContext, + tool: ToolInfo, + items: List[PolicyItemV2], + votes: int, +) -> List[PolicyItemV2]: + """Enrich every item in place and return them all, in order.""" + if not items: + return [] + + results = await asyncio.gather( + *[_enrich_item(llm, ctx, tool, item, votes) for item in items] + ) + + for item, reconciled in zip(items, results): + item.trigger = reconciled.trigger + item.requires = _validated_requires(reconciled, ctx, item.id) + item.references = _validated_references(reconciled, item) + item.pending_for_user = reconciled.pending_for_user + + if item.pending_for_user: + logger.info( + "{}: needs a human — {}", + item.id, + "; ".join(p.question for p in item.pending_for_user), + ) + + return items diff --git a/src/toolguard/buildtime/gen_spec_v2/stages/examples.py b/src/toolguard/buildtime/gen_spec_v2/stages/examples.py new file mode 100644 index 0000000..7dc251f --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/stages/examples.py @@ -0,0 +1,56 @@ +"""The ``examples`` stage: concrete compliance and violation scenarios. + +Examples are what the code generator's test generation works from, so they run +last — after ``enrich``, so an example for a ``post_tool`` rule can talk about +the result rather than the arguments. +""" + +import asyncio +from typing import List, Optional + +from toolguard.buildtime.gen_spec.data_types import ToolInfo +from toolguard.buildtime.gen_spec_v2 import prompts +from toolguard.buildtime.gen_spec_v2.context import GenContext +from toolguard.buildtime.gen_spec_v2.models import PolicyItemV2 +from toolguard.buildtime.gen_spec_v2.stages._shared import ( + item_summary, + messages, + str_list, +) +from toolguard.buildtime.llm import I_TG_LLM + + +async def run_examples( + llm: I_TG_LLM, + ctx: GenContext, + tool: ToolInfo, + items: List[PolicyItemV2], + example_number: Optional[int] = None, +) -> None: + """Write examples onto each item in place. + + ``example_number`` of ``None`` lets the model choose how many; a positive + number asks for exactly that many of each. + """ + if not items: + return + + system = prompts.system("examples") + + async def for_item(item: PolicyItemV2) -> None: + summary = item_summary(item) + summary["trigger"] = item.trigger.value + user = prompts.examples_user(ctx, tool, summary) + if example_number: + user += ( + f"\n\nWrite exactly {example_number} compliance example(s) and " + f"exactly {example_number} violation example(s)." + ) + + response = await llm.chat_json(messages(system, user)) + if not isinstance(response, dict): + return + item.compliance_examples = str_list(response.get("compliance_examples")) + item.violation_examples = str_list(response.get("violation_examples")) + + await asyncio.gather(*[for_item(item) for item in items]) diff --git a/src/toolguard/buildtime/gen_spec_v2/stages/expand.py b/src/toolguard/buildtime/gen_spec_v2/stages/expand.py new file mode 100644 index 0000000..c4b0f16 --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/stages/expand.py @@ -0,0 +1,67 @@ +"""The ``expand`` stage: repeated passes for rules ``create`` missed.""" + +from typing import List + +from loguru import logger + +from toolguard.buildtime.gen_spec.data_types import ToolInfo +from toolguard.buildtime.gen_spec_v2 import prompts +from toolguard.buildtime.gen_spec_v2.context import GenContext +from toolguard.buildtime.gen_spec_v2.models import PolicyItemV2, slugify +from toolguard.buildtime.gen_spec_v2.stages._shared import ( + build_item, + dict_list, + item_summary, + messages, +) +from toolguard.buildtime.llm import I_TG_LLM + + +async def run_expand( + llm: I_TG_LLM, + ctx: GenContext, + tool: ToolInfo, + items: List[PolicyItemV2], + iterations: int, +) -> List[PolicyItemV2]: + """Ask for additional items up to ``iterations`` times, stopping when dry. + + Each pass sees what is already captured, so it looks for gaps rather than + restating. A pass that adds nothing ends the loop: further passes over the + same inputs would only repeat it. + """ + items = list(items) + taken = {item.id for item in items} + + for iteration in range(max(0, iterations)): + response = await llm.chat_json( + messages( + prompts.system("expand"), + prompts.expand_user(ctx, tool, [item_summary(i) for i in items]), + ) + ) + + new_items: List[PolicyItemV2] = [] + for raw in dict_list(response, "policy_items"): + candidate_slug = slugify(raw.get("slug") or raw.get("name") or "") + if candidate_slug and f"{tool.name}.{candidate_slug}" in taken: + continue + item = build_item(raw, tool.name, taken) + if item is None: + continue + new_items.append(item) + + if not new_items: + logger.debug( + "expand({}): pass {} found nothing new, stopping", + tool.name, + iteration + 1, + ) + break + + items.extend(new_items) + logger.debug( + "expand({}): pass {} added {}", tool.name, iteration + 1, len(new_items) + ) + + return items diff --git a/src/toolguard/buildtime/gen_spec_v2/stages/review.py b/src/toolguard/buildtime/gen_spec_v2/stages/review.py new file mode 100644 index 0000000..e6344e8 --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/stages/review.py @@ -0,0 +1,84 @@ +"""The ``review`` stage: repeated relevance votes with a majority tally.""" + +import asyncio +from typing import Any, Dict, List, Sequence, Tuple + +from loguru import logger + +from toolguard.buildtime.gen_spec.data_types import ToolInfo +from toolguard.buildtime.gen_spec_v2 import prompts +from toolguard.buildtime.gen_spec_v2.context import GenContext +from toolguard.buildtime.gen_spec_v2.models import PolicyItemV2 +from toolguard.buildtime.gen_spec_v2.stages._shared import item_summary, messages +from toolguard.buildtime.llm import I_TG_LLM + + +def tally(votes: Sequence[Any]) -> Tuple[bool, str]: + """Keep the item when a strict majority of votes approve it on both counts. + + A vote missing either verdict counts against keeping: an item nobody could + confirm is relevant and checkable does not belong in the spec. + """ + usable = [v for v in votes if isinstance(v, dict)] + if not usable: + return False, "" + + approvals = sum( + 1 + for v in usable + if v.get("is_relevant") is True and v.get("can_be_validated") is True + ) + reasons = " ".join( + str(v.get("reason", "")).strip() for v in usable if v.get("reason") + ) + return approvals * 2 > len(usable), reasons + + +async def _review_item( + llm: I_TG_LLM, ctx: GenContext, tool: ToolInfo, item: PolicyItemV2, votes: int +) -> Tuple[bool, str]: + system = prompts.system("review") + user = prompts.review_user(ctx, tool, item_summary(item)) + results = await asyncio.gather( + *[llm.chat_json(messages(system, user)) for _ in range(votes)] + ) + return tally(list(results)) + + +async def run_review( + llm: I_TG_LLM, + ctx: GenContext, + tool: ToolInfo, + items: List[PolicyItemV2], + votes: int, +) -> Tuple[List[PolicyItemV2], List[Dict[str, Any]]]: + """Split ``items`` into survivors and archive entries. + + Archive entries keep the item's references so a later reader can see which + part of the policy document was considered and set aside. + """ + if not items: + return [], [] + + results = await asyncio.gather( + *[_review_item(llm, ctx, tool, item, votes) for item in items] + ) + + kept: List[PolicyItemV2] = [] + archived: List[Dict[str, Any]] = [] + for item, (keep, reason) in zip(items, results): + if keep: + kept.append(item) + continue + logger.debug("{}: archived by review — {}", item.id, reason) + archived.append( + { + "id": item.id, + "name": item.name, + "reason": reason, + "stage": "review", + "references": list(item.references), + } + ) + + return kept, archived diff --git a/src/toolguard/buildtime/gen_spec_v2/sysvars.py b/src/toolguard/buildtime/gen_spec_v2/sysvars.py new file mode 100644 index 0000000..b3890c5 --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/sysvars.py @@ -0,0 +1,118 @@ +"""The acting user's variables: loading, rendering, and validation. + +A policy like "HR may edit all employees' data" is only enforceable if the +runtime supplies the acting user's department. ``system_vars`` is how the +caller declares which such variables exist, so generation can reference them +by name instead of inventing them. + +Values may be any shape. A list is read as the closed set of allowed values, a +scalar as one example of the shape to expect, and a nested mapping or list is +kept and rendered as-is — a structured attribute of the acting user is still an +attribute of the acting user. + +Two keys are excluded: ``action_list`` and ``action_description``, which +describe the agent's own tools rather than the acting user, and appear in real +``system_vars.json`` files alongside the subject variables. +""" + +import json +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Union + +from loguru import logger +from pydantic import BaseModel, Field + +SUBJECT_PATH = "input.extensions.subject" + +NON_SUBJECT_KEYS = frozenset({"action_list", "action_description"}) +"""Keys that describe the agent's tools, not the acting user.""" + +SystemVarsInput = Union[Dict[str, Any], str, Path, None] + + +class SystemVars(BaseModel): + """The declared subject variables and their values.""" + + raw: Dict[str, Any] = Field(default_factory=dict) + + @property + def names(self) -> List[str]: + return list(self.raw) + + +def load_system_vars(source: SystemVarsInput) -> SystemVars: + """Load system variables from a dict, a path to ``sys_var.json``, or nothing. + + Values of any shape are kept, including nested ones. Only + :data:`NON_SUBJECT_KEYS` are dropped. Raises ``ValueError`` when a file does + not hold a JSON object, since a list or scalar has no variable names to + offer. + """ + if source is None: + return SystemVars() + + if isinstance(source, dict): + data: Dict[str, Any] = dict(source) + origin = "system_vars" + else: + path = Path(source) + data = json.loads(path.read_text(encoding="utf-8")) + origin = str(path) + if not isinstance(data, dict): + raise ValueError( + f"{path}: expected a JSON object of system variables, " + f"got {type(data).__name__}" + ) + + kept = {} + for name, value in data.items(): + if name in NON_SUBJECT_KEYS: + logger.debug( + "{}: ignoring '{}' — describes tools, not the user", origin, name + ) + continue + kept[name] = value + return SystemVars(raw=kept) + + +def render_system_vars(system_vars: SystemVars) -> str: + """Render each variable's name, access path, and domain, one per line. + + A list value is a closed set of allowed values; anything else is one example + of the shape to expect, nested structures included. The access path is + included so the LLM can tell subject variables from tool parameters. + """ + if not system_vars.raw: + return "(no system variables available)" + + lines = [] + for name, value in system_vars.raw.items(): + path = f"{SUBJECT_PATH}.{name}" + kind = "allowed values" if isinstance(value, list) else "example value" + lines.append(f"- {name} ({path}): {kind} = {json.dumps(value)}") + return "\n".join(lines) + + +def keep_declared( + names: Iterable[str], system_vars: SystemVars, context: Optional[str] = None +) -> List[str]: + """Keep only names that are actually declared, deduped, in first-seen order. + + A generated spec that references a variable nobody supplies would compile + into a guard reading an absent value, so an undeclared name is dropped and + logged rather than trusted. + """ + declared = system_vars.raw + kept: List[str] = [] + for name in names: + if name in kept: + continue + if name not in declared: + logger.warning( + "Dropping undeclared system variable '{}'{}", + name, + f" in {context}" if context else "", + ) + continue + kept.append(name) + return kept diff --git a/src/toolguard/buildtime/gen_spec_v2/tools_input.py b/src/toolguard/buildtime/gen_spec_v2/tools_input.py new file mode 100644 index 0000000..75202d8 --- /dev/null +++ b/src/toolguard/buildtime/gen_spec_v2/tools_input.py @@ -0,0 +1,45 @@ +"""Normalizing every accepted tool-description shape into ``ToolInfo``. + +v2 accepts what v1 accepts — callables or an OpenAPI dict — plus a plain +``list[ToolInfo]``, which is what a caller holding MCP tool definitions +already has. v1's ``_tools_to_tool_infos`` documents that branch but raises +on it; implementing it here keeps v1 untouched. +""" + +from typing import Any, Callable, Dict, List, Sequence, Union, cast + +from toolguard.buildtime.gen_spec.data_types import ToolInfo +from toolguard.buildtime.gen_spec.fn_to_toolinfo import function_to_toolInfo +from toolguard.buildtime.gen_spec.oas_to_toolinfo import openapi_to_toolinfos +from toolguard.buildtime.utils.open_api import OpenAPI + +TOOLS_V2 = Union[Dict[str, Any], Sequence[Union[Callable, ToolInfo]]] + + +def to_tool_infos(tools: TOOLS_V2) -> List[ToolInfo]: + """Convert ``tools`` to ``ToolInfo``s, preserving the caller's order. + + Accepts an OpenAPI spec dict, or a sequence mixing callables and + ``ToolInfo``s. Raises ``ValueError`` on an empty sequence — generating + specs for no tools is always a caller mistake — and + ``NotImplementedError`` for an element that is neither. + """ + if isinstance(tools, dict): + return openapi_to_toolinfos(OpenAPI.model_validate(tools)) + + if isinstance(tools, (list, tuple)): + if not tools: + raise ValueError("No tools supplied") + infos: List[ToolInfo] = [] + for tool in tools: + if isinstance(tool, ToolInfo): + infos.append(tool) + elif callable(tool): + infos.append(function_to_toolInfo(cast(Callable, tool))) + else: + raise NotImplementedError( + f"Unsupported tool description: {type(tool).__name__}" + ) + return infos + + raise NotImplementedError(f"Unsupported tools input: {type(tools).__name__}") diff --git a/tests/buildtime/e2e/__init__.py b/tests/buildtime/e2e/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/buildtime/e2e/test_gen_spec_v2_codegen.py b/tests/buildtime/e2e/test_gen_spec_v2_codegen.py new file mode 100644 index 0000000..16e7c68 --- /dev/null +++ b/tests/buildtime/e2e/test_gen_spec_v2_codegen.py @@ -0,0 +1,197 @@ +"""End to end: v2 specs -> adapter -> generated guards that actually run. + +This is the acceptance test for the adapter. The `gen_spec_v2` unit tests prove +the spec shape and the `skip` classification; only these tests prove that what +comes out the other side still drives `gen_py` and produces guards the runtime +enforces. + +Each test mirrors one variant of `test_calculator.py` — the same policy, the same +tool-input shape, and the same runtime assertion battery — so "does v2 cover what +v1 covers" is answered by the two files having the same variants. + +Needs LLM credentials and is skipped without them. The credential-free half of +the same contract is in `tests/buildtime/gen_spec_v2/test_gen_py_contract.py`. +""" + +import os +import shutil +from pathlib import Path +from typing import Any, Dict, Optional, Type, TypeVar + +import pytest +from dotenv import load_dotenv +from examples.calculator.inputs import tool_functions as fn_tools +from examples.calculator.inputs import tool_langchain as lg_tools +from examples.calculator.inputs import tool_methods as mtd_tools + +from toolguard.buildtime import ( + LitellmModel, + SpecV2Options, + generate_guard_specs_v2_full, + generate_guards_code, + specs_v2_to_v1, +) +from toolguard.buildtime.data_types import TOOLS +from toolguard.buildtime.llm import I_TG_LLM +from toolguard.buildtime.utils.open_api import OpenAPI +from toolguard.extra.api_to_functions import api_cls_to_functions +from toolguard.extra.langchain_to_oas import langchain_tools_to_openapi +from toolguard.runtime import ( + IToolInvoker, + LangchainToolInvoker, + ToolFunctionsInvoker, + ToolMethodsInvoker, +) + +# The v1 calculator e2e owns the runtime assertion battery; importing it keeps +# the two suites asserting exactly the same enforcement. +from .test_calculator import assert_toolgurards_run + +POLICY_PATH = Path("tests/examples/calculator/inputs/policy_doc.md") +WORK_ROOT = Path("tests/tmp/e2e/gen_spec_v2") + +# conftest's autouse fixture loads .env too late for a collection-time skipif, +# so load it here before the condition is evaluated. +load_dotenv() + +requires_llm = pytest.mark.skipif( + not os.getenv("LLM_API_KEY"), + reason="needs LLM credentials (LLM_API_KEY)", +) + +# Fewer votes and passes than the defaults: these tests exist to prove the +# v2 -> adapter -> codegen path, not to tune generation quality. +FAST = SpecV2Options(add_iterations=1, review_votes=3, example_number=2) + + +def llm() -> I_TG_LLM: + return LitellmModel( + model_name=os.getenv("MODEL_NAME") or "gpt-4o-2024-08-06", + provider=os.getenv("LLM_PROVIDER") or "azure", + kw_args={ + "api_base": os.getenv("LLM_API_BASE"), + "api_version": os.getenv("LLM_API_VERSION"), + "api_key": os.getenv("LLM_API_KEY"), + }, + ) + + +async def _build_v2_guards( + variant: str, + tools: TOOLS, + known_tools: list, + options: Optional[SpecV2Options] = None, +): + """v2 specs -> adapter -> generated guard code, for one tool-input shape. + + Raw markdown, not v1's markdown->HTML conversion: v2's reference grounding + strips markdown emphasis and segments on `- ` bullets, so HTML input yields + references like "

Division by Zero..." instead of the rule text. + """ + policy_text = POLICY_PATH.read_text(encoding="utf-8") + + work_dir = WORK_ROOT / variant + shutil.rmtree(work_dir, ignore_errors=True) + + specs = await generate_guard_specs_v2_full( + policy_text, + tools, + llm(), + work_dir / "specs_v2", + source_doc=str(POLICY_PATH), + options=options or FAST, + ) + + assert specs, "no v2 specs generated" + assert all(item.references for spec in specs for item in spec.policy_items), ( + "every item must be grounded in the policy document" + ) + assert all( + item.id.startswith(spec.tool_name + ".") + for spec in specs + for item in spec.policy_items + ), "every item id must be namespaced by its tool" + + v1_specs = specs_v2_to_v1(specs, known_tools=known_tools) + assert any(not item.skip for spec in v1_specs for item in spec.policy_items), ( + "the calculator policy is argument-only, so something must be codegen-able" + ) + + return await generate_guards_code( + tool_specs=v1_specs, + tools=tools, + work_dir=work_dir / "code", + llm=llm(), + app_name=f"calc_v2_{variant}", + ) + + +CALC_FUNCS = [ + fn_tools.divide_tool, + fn_tools.add_tool, + fn_tools.subtract_tool, + fn_tools.multiply_tool, + fn_tools.map_kdi_number, +] + + +@requires_llm +async def test_tool_functions(): + guards = await _build_v2_guards( + "fns", CALC_FUNCS, [fn.__name__ for fn in CALC_FUNCS] + ) + + await assert_toolgurards_run(guards, ToolFunctionsInvoker(CALC_FUNCS)) + + +@requires_llm +async def test_tool_methods(): + fns = api_cls_to_functions(mtd_tools.CalculatorTools) + + guards = await _build_v2_guards("mtds", fns, [fn.__name__ for fn in fns]) + + await assert_toolgurards_run( + guards, ToolMethodsInvoker(mtd_tools.CalculatorTools()) + ) + + +@requires_llm +async def test_tools_langchain(): + tools = [ + lg_tools.divide_tool, + lg_tools.add_tool, + lg_tools.subtract_tool, + lg_tools.multiply_tool, + lg_tools.map_kdi_number, + ] + oas = langchain_tools_to_openapi(tools) + + guards = await _build_v2_guards("lg", oas, [tool.name for tool in tools]) + + # openapi_spec=True: OpenAPI-shaped tools take their arguments wrapped. + await assert_toolgurards_run(guards, LangchainToolInvoker(tools), True) + + +@requires_llm +async def test_tools_openapi_spec(): + oas = OpenAPI.load_from("tests/examples/calculator/inputs/oas.json") + oas_dict = oas.model_dump() + + guards = await _build_v2_guards("oas", oas_dict, [fn.__name__ for fn in CALC_FUNCS]) + + class DummyInvoker(IToolInvoker): + """Compute inline instead of calling a remote web method.""" + + T = TypeVar("T") + + def __init__(self) -> None: + self._funcs_by_name = {fn.__name__: fn for fn in CALC_FUNCS} + + async def invoke( + self, toolname: str, arguments: Dict[str, Any], return_type: Type[T] + ) -> T: + func = self._funcs_by_name.get(toolname) + assert callable(func), f"Tool {toolname} was not found" + return func(**arguments) # type: ignore[return-value] + + await assert_toolgurards_run(guards, DummyInvoker(), True) diff --git a/tests/buildtime/e2e/test_guard_set_delta.py b/tests/buildtime/e2e/test_guard_set_delta.py new file mode 100644 index 0000000..4bc12ed --- /dev/null +++ b/tests/buildtime/e2e/test_guard_set_delta.py @@ -0,0 +1,175 @@ +"""What guards v1 produces versus v2-via-adapter, over identical inputs. + +Neither generator is "correct" by definition, so this reports rather than +asserts: it runs both step-1 pipelines on the same policy and tools, computes +the set of items each would hand to codegen, and writes the difference to a +markdown file. The point is to make the migration cost visible — v2 has a +different prompt lineage and a different stage list, so its guard set is not +expected to match v1's. + +Reading the report: +- **v1 only** — a guard you would lose by switching. Worth understanding. +- **v2 only** — a rule v1 missed or deleted. +- **skipped by v2** — rules v2 captured but that today's codegen cannot enforce + (identity, conversation, post-call, or an open question). These are the honest + gap, not a regression: v1 dropped them silently. + +Needs LLM credentials and runs two full spec generations, so it is opt-in via +`-m delta`. +""" + +import os +from pathlib import Path +from typing import Dict, List + +import pytest +from dotenv import load_dotenv +from examples.calculator.inputs import tool_functions as fn_tools + +from toolguard.buildtime import ( + LitellmModel, + SpecV2Options, + generate_guard_specs, + generate_guard_specs_v2_full, + specs_v2_to_v1, +) +from toolguard.buildtime.gen_spec_v2.models import slugify +from toolguard.buildtime.llm import I_TG_LLM + +load_dotenv() + +POLICY_PATH = Path("tests/examples/calculator/inputs/policy_doc.md") +WORK_ROOT = Path("tests/tmp/e2e/guard_set_delta") +REPORT_PATH = WORK_ROOT / "guard_set_delta.md" + +requires_llm = pytest.mark.skipif( + not os.getenv("LLM_API_KEY"), + reason="needs LLM credentials (LLM_API_KEY)", +) + +CALC_FUNCS = [ + fn_tools.divide_tool, + fn_tools.add_tool, + fn_tools.subtract_tool, + fn_tools.multiply_tool, + fn_tools.map_kdi_number, +] + + +def llm() -> I_TG_LLM: + return LitellmModel( + model_name=os.getenv("MODEL_NAME") or "gpt-4o-2024-08-06", + provider=os.getenv("LLM_PROVIDER") or "azure", + kw_args={ + "api_base": os.getenv("LLM_API_BASE"), + "api_version": os.getenv("LLM_API_VERSION"), + "api_key": os.getenv("LLM_API_KEY"), + }, + ) + + +def _guard_set(specs) -> Dict[str, List[str]]: + """Tool -> names of items codegen would receive (i.e. not skipped).""" + return { + spec.tool_name: sorted(i.name for i in spec.policy_items if not i.skip) + for spec in specs + } + + +def _skipped(specs) -> Dict[str, List[str]]: + return { + spec.tool_name: sorted( + f"{i.name} — {i.debug.get('skip_reason', 'skipped')}" + for i in spec.policy_items + if i.skip + ) + for spec in specs + } + + +def _render(v1: Dict[str, List[str]], v2: Dict[str, List[str]], skipped) -> str: + """Render the comparison, matching rules by slug rather than exact name. + + The two generators word an item's `name` differently for the same rule — + v1 tends to copy the policy heading, v2 writes a sentence — so comparing + raw names puts every rule in both "only" columns and reads as though every + guard were lost and replaced. Slugs collapse that cosmetic difference so the + columns show rules that genuinely exist on one side only. + """ + lines = [ + "# Guard-set delta: v1 vs v2-via-adapter", + "", + f"Policy: `{POLICY_PATH}` ", + f"Model: `{os.getenv('MODEL_NAME')}`", + "", + "Rules are matched by slug, so a rule both generators found is not listed as", + '"only" just because they worded its name differently.', + "", + "| Tool | v1 guards | v2 guards | v1 only | v2 only |", + "|---|---|---|---|---|", + ] + for tool in sorted(set(v1) | set(v2)): + a = {slugify(name): name for name in v1.get(tool, [])} + b = {slugify(name): name for name in v2.get(tool, [])} + only_a = [a[s] for s in sorted(set(a) - set(b))] + only_b = [b[s] for s in sorted(set(b) - set(a))] + lines.append( + f"| `{tool}` | {len(a)} | {len(b)} | " + f"{'; '.join(only_a) or '—'} | {'; '.join(only_b) or '—'} |" + ) + + lines += ["", "## Captured by v2 but not enforceable today", ""] + any_skipped = False + for tool in sorted(skipped): + for entry in skipped[tool]: + any_skipped = True + lines.append(f"- `{tool}`: {entry}") + if not any_skipped: + lines.append("_none_") + + lines += [ + "", + "## Totals", + "", + f"- v1 guards: {sum(len(v) for v in v1.values())}", + f"- v2 guards: {sum(len(v) for v in v2.values())}", + f"- v2 captured but not enforceable: {sum(len(v) for v in skipped.values())}", + "", + ] + return "\n".join(lines) + + +@pytest.mark.delta +@requires_llm +async def test_report_guard_set_delta(): + policy_markdown = POLICY_PATH.read_text(encoding="utf-8") + WORK_ROOT.mkdir(parents=True, exist_ok=True) + + v1_specs = await generate_guard_specs( + policy_text=policy_markdown, + tools=CALC_FUNCS, + llm=llm(), + work_dir=WORK_ROOT / "v1", + ) + + v2_specs = await generate_guard_specs_v2_full( + policy_markdown, + CALC_FUNCS, + llm(), + WORK_ROOT / "v2", + source_doc=str(POLICY_PATH), + options=SpecV2Options(), + ) + v2_as_v1 = specs_v2_to_v1(v2_specs, known_tools=[fn.__name__ for fn in CALC_FUNCS]) + + v1_set, v2_set = _guard_set(v1_specs), _guard_set(v2_as_v1) + report = _render(v1_set, v2_set, _skipped(v2_as_v1)) + REPORT_PATH.write_text(report, encoding="utf-8") + print("\n" + report) + + # Reporting, not asserting equality: the only hard requirement is that + # neither generator came back empty, which would mean the run failed rather + # than that the generators disagree. + assert sum(len(v) for v in v1_set.values()) > 0, "v1 produced no guards" + assert sum(len(v) for v in v2_set.values()) > 0, "v2 produced no guards" + assert REPORT_PATH.exists() diff --git a/tests/buildtime/e2e/test_tau2_v2.py b/tests/buildtime/e2e/test_tau2_v2.py new file mode 100644 index 0000000..55748ab --- /dev/null +++ b/tests/buildtime/e2e/test_tau2_v2.py @@ -0,0 +1,264 @@ +"""tau2 airline, generated through v2 specs instead of v1. + +The v2 counterpart of `test_tau2.py`. This is the only e2e over a realistic +domain — 40-odd airline tools rather than five calculator functions — so it is +where v2's per-tool binding has to hold up: a one-sentence policy must attach to +the two tools it governs and to no others. + +The `complex_api` case is the more interesting one for v2: the cancellation rule +cannot be decided from arguments alone, so v2 should declare a `tool_history` +requirement (a `get_reservation_details` lookup) and still be codegen-able, +because generated guards do get an `api` handle. + +Needs LLM credentials and the tau2 package; skipped without either. +""" + +import os +import shutil +import unittest +from datetime import datetime, timedelta +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +import pytest +from dotenv import load_dotenv + +from toolguard.buildtime import ( + I_TG_LLM, + SpecV2Options, + generate_guard_specs_v2_full, + generate_guards_code, + specs_v2_to_v1, +) +from toolguard.buildtime.llm.tg_litellm import LitellmModel +from toolguard.extra import api_cls_to_functions +from toolguard.runtime import load_toolguards +from toolguard.runtime.data_types import PolicyViolationException +from toolguard.runtime.tool_invokers.methods import ToolMethodsInvoker + +load_dotenv() + +tau2 = pytest.importorskip("tau2", reason="needs the tau2 package") +from tau2.domains.airline.data_model import ( # noqa: E402 + CabinClass, + Insurance, + Passenger, + Payment, + Reservation, + ReservationFlight, +) +from tau2.domains.airline.tools import AirlineTools # noqa: E402 + +WORK_ROOT = Path("tests/tmp/e2e/gen_spec_v2_tau2") + +requires_llm = pytest.mark.skipif( + not os.getenv("LLM_API_KEY"), + reason="needs LLM credentials (LLM_API_KEY)", +) + +# review_votes stays at the default 5 here, unlike the calculator e2e: with 40-odd +# airline tools there are many chances to bind a rule to a tool it does not govern, +# and the relevance vote is what has to reject those. +# +# on_tool_error="raise" so a model failure for one tool surfaces as that error rather +# than as a confusing "13 specs for 14 tools" count mismatch downstream. With 40-odd +# tools a transient unparseable-JSON response is not rare, and it must not be +# mistaken for a policy-binding defect. +FAST = SpecV2Options( + add_iterations=1, review_votes=5, example_number=2, on_tool_error="raise" +) + + +def llm() -> I_TG_LLM: + return LitellmModel( + model_name=os.getenv("MODEL_NAME") or "gpt-4o-2024-08-06", + provider=os.getenv("LLM_PROVIDER") or "azure", + kw_args={ + "api_base": os.getenv("LLM_API_BASE"), + "api_version": os.getenv("LLM_API_VERSION"), + "api_key": os.getenv("LLM_API_KEY"), + }, + ) + + +def _airline_tools(): + fns = api_cls_to_functions(AirlineTools) + return [fn for fn in fns if hasattr(fn, "__tool__")] + + +def _passengers(n: int): + return n * [Passenger(first_name="John", last_name="Doe", dob="1990-01-01")] + + +async def _build(variant: str, policy_text: str, app_name: str, tool_fns): + work_dir = WORK_ROOT / variant + shutil.rmtree(work_dir, ignore_errors=True) + + specs = await generate_guard_specs_v2_full( + policy_text, + tool_fns, + llm(), + work_dir / "specs_v2", + source_doc="inline policy", + options=FAST, + ) + + assert len(specs) == len(tool_fns), "one spec per tool, empty ones included" + + v1_specs = specs_v2_to_v1(specs, known_tools=[fn.__name__ for fn in tool_fns]) + guards = await generate_guards_code( + tool_specs=v1_specs, + tools=tool_fns, + work_dir=work_dir / "code", + llm=llm(), + app_name=app_name, + ) + return specs, v1_specs, guards + + +def _spec_for(specs, tool_name): + return next(spec for spec in specs if spec.tool_name == tool_name) + + +@requires_llm +async def test_tau2_simple(): + tool_fns = _airline_tools() + policy_text = "Users cannot book a flight for more than 5 passengers" + + specs, v1_specs, guards = await _build( + "simple", policy_text, "tau2_v2_simple", tool_fns + ) + + governed = ("book_reservation", "update_reservation_passengers") + for name in governed: + spec = _spec_for(specs, name) + assert len(spec.policy_items) >= 1, f"{name} must be governed by the policy" + item = spec.policy_items[0] + assert item.compliance_examples and item.violation_examples + assert item.trigger == "pre_tool" + + # The policy names one condition on one argument, so it must not spread to + # tools it does not govern. + for spec in specs: + if spec.tool_name not in governed: + assert not spec.policy_items, f"{spec.tool_name} should be ungoverned" + + # A passenger-count rule needs nothing but the arguments, so it must survive + # the adapter — otherwise no guard is generated at all. + for name in governed: + v1_spec = _spec_for(v1_specs, name) + assert any(not item.skip for item in v1_spec.policy_items), ( + f"{name}'s rule is argument-only and must be codegen-able" + ) + + api = MagicMock() + with load_toolguards(guards.out_dir) as toolguard: + for name in governed: + await toolguard.guard_toolcall( + name, {"passengers": _passengers(5)}, ToolMethodsInvoker(api) + ) + for name in governed: + with unittest.TestCase().assertRaises(PolicyViolationException): + await toolguard.guard_toolcall( + name, {"passengers": _passengers(6)}, ToolMethodsInvoker(api) + ) + + +def _reservation( + cabin: CabinClass, insurance: Insurance, days_ago: int = 14 +) -> Reservation: + created_at = (datetime.now() - timedelta(days=days_ago)).strftime( + "%Y-%m-%dT%H:%M:%S" + ) + return Reservation( + reservation_id="ZFA04Y", + user_id="sara_doe_496", + origin="SFO", + destination="JFK", + flight_type="round_trip", + cabin=cabin, + flights=[ + ReservationFlight( + flight_number="HAT001", + origin="SFO", + destination="JFK", + date="2024-06-15", + price=1200, + ) + ], + passengers=_passengers(1), + payment_history=[Payment(payment_id="pay_001", amount=1200)], + created_at=created_at, + total_baggages=1, + nonfree_baggages=0, + insurance=insurance, + ) + + +def _api_for(reservation: Reservation) -> MagicMock: + api = MagicMock() + api.get_reservation_details = AsyncMock() + api.get_reservation_details.side_effect = ( + lambda reservation_id: reservation if reservation_id == "ZFA04Y" else None + ) + api.get_flight_status = AsyncMock() + api.get_flight_status.side_effect = ( + lambda flight_number, date: "scheduled" if flight_number == "HAT001" else None + ) + return api + + +@requires_llm +async def test_tau2_complex_api(): + tool_fns = _airline_tools() + policy_text = """To use the 'cancel_reservation' tool, at least one the following must hold: + 1) The cancellation is within 24 hours of booking, + 2) The airline cancelled the flight, + 3) For economy class, cancellation is only allowed if travel insurance was purchased and qualifies, + 4) Business class flights can be cancelled anytime. + These conditions must be validated prior to invoking the tool. + """ + + specs, v1_specs, guards = await _build( + "complex_api", policy_text, "tau2_v2_api", tool_fns + ) + + cancel_spec = _spec_for(specs, "cancel_reservation") + assert len(cancel_spec.policy_items) >= 1 + item = cancel_spec.policy_items[0] + assert item.compliance_examples and item.violation_examples + # "validated prior to invoking the tool" is explicit in the policy. + assert item.trigger == "pre_tool" + # The cabin, booking time and insurance all live on the reservation, not in + # the arguments, so the rule has to declare the lookup it depends on. + assert item.requires.tool_history, ( + "cancellation depends on the reservation, so a tool lookup must be declared" + ) + + for spec in specs: + if spec.tool_name != "cancel_reservation": + assert not spec.policy_items, f"{spec.tool_name} should be ungoverned" + + assert any( + not item.skip for item in _spec_for(v1_specs, "cancel_reservation").policy_items + ), "a tool_history-only rule must remain codegen-able" + + # Business class can be cancelled anytime. + with load_toolguards(guards.out_dir) as toolguard: + await toolguard.guard_toolcall( + "cancel_reservation", + args={"reservation_id": "ZFA04Y"}, + delegate=ToolMethodsInvoker(_api_for(_reservation("business", "no"))), + ) + + # Basic economy without insurance, booked two weeks ago: none of the four + # conditions hold. + with load_toolguards(guards.out_dir) as toolguard: + with unittest.TestCase().assertRaises(PolicyViolationException): + await toolguard.guard_toolcall( + "cancel_reservation", + args={"reservation_id": "ZFA04Y"}, + delegate=ToolMethodsInvoker( + _api_for(_reservation("basic_economy", "no")) + ), + ) diff --git a/tests/buildtime/gen_spec_v2/__init__.py b/tests/buildtime/gen_spec_v2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/buildtime/gen_spec_v2/conftest.py b/tests/buildtime/gen_spec_v2/conftest.py new file mode 100644 index 0000000..80993f4 --- /dev/null +++ b/tests/buildtime/gen_spec_v2/conftest.py @@ -0,0 +1,113 @@ +"""A scripted LLM for the stage and pipeline tests. + +Responses are keyed by the ``[STAGE:x]`` marker every v2 user prompt carries, +so a test declares what each stage answers without caring about call order or +concurrency. +""" + +import re +from pathlib import Path +from typing import Any, Callable, Dict, List, Union + +import pytest + +from toolguard.buildtime.gen_spec.data_types import ToolInfo, ToolInfoParam +from toolguard.buildtime.gen_spec_v2.context import GenContext +from toolguard.buildtime.llm import I_TG_LLM + +CORPUS_DIR = Path("tests/data/specs_v2") +"""Ground-truth employee specs. Not committed; see the test-baseline doc.""" + +CORPUS_INPUTS_DIR = Path("tests/data/specs_v2_inputs") +"""The policy document, tool definitions and system vars those specs came from.""" + +requires_corpus = pytest.mark.skipif( + not CORPUS_DIR.is_dir() or not CORPUS_INPUTS_DIR.is_dir(), + reason=( + "needs the employee ground-truth corpus under tests/data/ — copy " + "smith/examples/employee/smith/{smith_outputs/ground_truth_specs,guidance.txt," + "system_vars.json,tool_definitions.json} into tests/data/specs_v2{,_inputs}/" + ), +) +"""Skip a test that reads the ground-truth corpus, when it is not present. + +Every test that needs the corpus carries this, and every module that reads it at +import time guards that read, so a checkout without `tests/data/` still collects +and runs the rest of the suite instead of failing collection outright. +""" + +STAGE_MARKER = re.compile(r"\[STAGE:(\w+)\]") + +Response = Union[Dict[str, Any], Callable[[str], Dict[str, Any]], Exception] + + +class FakeLLM(I_TG_LLM): + """Returns a canned response per stage, and records every call.""" + + def __init__(self, responses: Dict[str, Response]): + self.responses = responses + self.calls: List[Dict[str, str]] = [] + + async def chat_json(self, messages: List[Dict]) -> Dict: + user_content = messages[-1]["content"] + match = STAGE_MARKER.search(user_content) + stage = match.group(1) if match else "unknown" + self.calls.append({"stage": stage, "content": user_content}) + + response = self.responses.get(stage, {}) + if isinstance(response, Exception): + raise response + if callable(response): + return response(user_content) + return response + + async def generate(self, messages: List[Dict]) -> str: + raise NotImplementedError + + def calls_for(self, stage: str) -> List[Dict[str, str]]: + return [call for call in self.calls if call["stage"] == stage] + + def count(self, stage: str) -> int: + return len(self.calls_for(stage)) + + +POLICY = """# Employee Policy + +- **HR** may view and edit all employees' data. +- An employee's `salary` may be updated only by **HR** or by that employee's **direct manager**. +- When an employee's **salary** is set or updated, it must be a positive amount (greater than zero). +""" + +SYSTEM_VARS = { + "user_id": 1, + "department": ["Corporate Leadership", "Engineering", "HR", "Finance"], + "organization": ["IBM Corporation", "Red Hat", "Kyndryl"], +} + + +def make_tool(name: str, description: str = "does a thing") -> ToolInfo: + return ToolInfo( + name=name, + summary="", + description=description, + parameters={ + "user_id": ToolInfoParam(type="int", description="who", required=True), + "salary": ToolInfoParam(type="float", description="pay", required=False), + }, + signature=f"{name}(user_id: int, salary: float) -> dict", + ) + + +@pytest.fixture +def tool() -> ToolInfo: + return make_tool("update_employee", "Update an employee") + + +@pytest.fixture +def other_tool() -> ToolInfo: + return make_tool("get_employee", "Read an employee") + + +@pytest.fixture +def ctx(tool, other_tool) -> GenContext: + return GenContext.build(POLICY, [tool, other_tool], SYSTEM_VARS) diff --git a/tests/buildtime/gen_spec_v2/test_adapter.py b/tests/buildtime/gen_spec_v2/test_adapter.py new file mode 100644 index 0000000..e48fa21 --- /dev/null +++ b/tests/buildtime/gen_spec_v2/test_adapter.py @@ -0,0 +1,186 @@ +"""Converting a v2 spec into the v1 spec that `gen_py` consumes. + +`skip` is the whole point: it marks the items today's codegen and runtime +cannot enforce *correctly*. Generated guards receive `args` + `api` only — +no acting user, no chat history — and there is no post-invocation hook, so a +guard emitted for one of those rules would check the wrong thing rather than +nothing. +""" + +from pathlib import Path + +import pytest + +from toolguard.buildtime.gen_spec_v2.adapter import spec_v2_to_v1, specs_v2_to_v1 +from toolguard.buildtime.gen_spec_v2.models import ( + PendingItem, + PendingType, + PolicyItemV2, + Requires, + SpecToolInfo, + SpecDebugV2, + SpecV2, + ToolHistoryEntry, + Trigger, +) +from toolguard.buildtime.gen_spec_v2.serialize import load_spec +from toolguard.runtime.data_types import ToolGuardSpec + +from .conftest import requires_corpus + +SPEC_DIR = Path("tests/data/specs_v2") + + +def _item(name="a rule", **kwargs) -> PolicyItemV2: + return PolicyItemV2( + id=kwargs.pop("id", "update_employee.a_rule"), + name=name, + description="the rule", + compliance_examples=["ok"], + violation_examples=["bad"], + references=["the source line"], + **kwargs, + ) + + +def _spec(*items: PolicyItemV2) -> SpecV2: + return SpecV2( + tool_name="update_employee", + source_doc="policy.md", + policy_items=list(items), + debug=SpecDebugV2(tool_info=SpecToolInfo(is_read_only=False)), + ) + + +# --- skip truth table ------------------------------------------------------ + + +def test_pure_argument_rule_is_not_skipped(): + v1 = spec_v2_to_v1(_spec(_item())) + + assert v1.policy_items[0].skip is False + + +def test_rule_needing_system_vars_is_skipped(): + item = _item(requires=Requires(system_vars=["department"])) + + assert spec_v2_to_v1(_spec(item)).policy_items[0].skip is True + + +def test_rule_needing_message_history_is_skipped(): + item = _item(requires=Requires(message_history=True)) + + assert spec_v2_to_v1(_spec(item)).policy_items[0].skip is True + + +def test_post_tool_rule_is_skipped(): + item = _item(trigger=Trigger.post_tool) + + assert spec_v2_to_v1(_spec(item)).policy_items[0].skip is True + + +def test_rule_with_a_pending_gap_is_skipped(): + item = _item( + pending_for_user=[ + PendingItem( + type=PendingType.missing_var, detail="no blacklist", question="where?" + ) + ] + ) + + assert spec_v2_to_v1(_spec(item)).policy_items[0].skip is True + + +def test_rule_needing_only_a_prior_tool_call_is_not_skipped(): + # tool_history alone is enforceable today: generated guards get `api`. + item = _item( + requires=Requires(tool_history=[ToolHistoryEntry(tool="get_employee")]) + ) + + assert spec_v2_to_v1(_spec(item)).policy_items[0].skip is False + + +# --- fields gen_py depends on --------------------------------------------- + + +def test_content_fields_survive_the_conversion(): + v1_item = spec_v2_to_v1(_spec(_item())).policy_items[0] + + assert v1_item.name == "a rule" + assert v1_item.description == "the rule" + assert v1_item.references == ["the source line"] + assert v1_item.compliance_examples == ["ok"] + assert v1_item.violation_examples == ["bad"] + + +def test_colliding_item_names_are_disambiguated(): + # gen_py derives one module, function, and test file per item.name, so two + # items sharing a name would overwrite each other's files. + spec = _spec( + _item(name="same", id="update_employee.first"), + _item(name="same", id="update_employee.second"), + ) + + names = [i.name for i in spec_v2_to_v1(spec).policy_items] + + assert len(set(names)) == 2 + assert names[0] == "same" + assert "second" in names[1] + + +def test_v2_only_fields_are_preserved_in_debug(): + item = _item( + trigger=Trigger.post_tool, + requires=Requires(system_vars=["user_id"]), + ) + + v1_item = spec_v2_to_v1(_spec(item)).policy_items[0] + + assert v1_item.debug["id"] == "update_employee.a_rule" + assert v1_item.debug["trigger"] == "post_tool" + assert v1_item.debug["requires"]["system_vars"] == ["user_id"] + + +def test_spec_level_fields_are_preserved_in_debug(): + v1 = spec_v2_to_v1(_spec(_item())) + + assert v1.tool_name == "update_employee" + assert v1.debug["source_doc"] == "policy.md" + assert v1.debug["tool_info"]["is_read_only"] is False + + +# --- the real corpus ------------------------------------------------------- + + +@requires_corpus +@pytest.mark.parametrize("path", sorted(SPEC_DIR.glob("*.json")), ids=lambda p: p.stem) +def test_every_ground_truth_spec_converts_and_validates(path: Path): + v1 = spec_v2_to_v1(load_spec(path)) + + # The output must survive v1's own loader, which is what gen_py gets. + assert ToolGuardSpec.model_validate(v1.model_dump()).tool_name == v1.tool_name + + +@requires_corpus +def test_employee_corpus_skips_every_identity_dependent_rule(): + v1 = spec_v2_to_v1(load_spec(SPEC_DIR / "update_employee.json")) + unskipped = {i.name for i in v1.policy_items if not i.skip} + + assert "Salary must be positive when set or updated" in unskipped + assert "Editing an employee record is limited to self or HR" not in unskipped + + +def test_specs_for_unknown_tools_are_dropped(): + # gen_py would otherwise try to generate a guard for a tool that does not + # exist (this is how a `global`-style spec used to leak through). + specs = [_spec(_item()), SpecV2(tool_name="global", policy_items=[_item()])] + + converted = specs_v2_to_v1(specs, known_tools=["update_employee"]) + + assert [s.tool_name for s in converted] == ["update_employee"] + + +def test_all_tools_are_kept_when_no_tool_list_is_given(): + specs = [_spec(_item()), SpecV2(tool_name="other", policy_items=[_item()])] + + assert len(specs_v2_to_v1(specs)) == 2 diff --git a/tests/buildtime/gen_spec_v2/test_conflicts.py b/tests/buildtime/gen_spec_v2/test_conflicts.py new file mode 100644 index 0000000..c8a51f3 --- /dev/null +++ b/tests/buildtime/gen_spec_v2/test_conflicts.py @@ -0,0 +1,177 @@ +"""Conflict detection within a tool, and routing conflicts onto specs. + +Routing is the part that differs from smith: with no `global.json`, a conflict +spanning several tools attaches to each involved tool's spec so every spec +stays self-contained. +""" + +from toolguard.buildtime.gen_spec_v2.conflicts import ( + attach_conflicts, + find_conflicts, +) +from toolguard.buildtime.gen_spec_v2.models import Conflict, PolicyItemV2, SpecV2 + +from .conftest import FakeLLM + + +def _item(id_: str, name="a rule") -> PolicyItemV2: + return PolicyItemV2(id=id_, name=name, description="d", references=["r"]) + + +def _spec(tool: str, *ids: str) -> SpecV2: + return SpecV2(tool_name=tool, policy_items=[_item(i) for i in ids]) + + +def _raw_conflict( + id_="conflict.update_employee.scope", policies=("update_employee.a",) +): + return { + "id": id_, + "name": "a conflict", + "kind": "scope", + "conflicting_policies": list(policies), + "description": "when both apply", + "question": "which wins?", + } + + +# --- find_conflicts -------------------------------------------------------- + + +async def test_a_tool_with_one_item_is_not_examined(): + # There is no pair to compare, so no call should be made. + llm = FakeLLM({"conflicts": {"conflicts": []}}) + + await find_conflicts(llm, [_spec("update_employee", "update_employee.a")]) + + assert llm.count("conflicts") == 0 + + +async def test_pairwise_calls_are_upper_triangle_only(): + # 4 items -> compare item i against items i+1.. for i in 0..2 = 3 calls, + # not 12: each prompt stays bounded by one tool's item count. + llm = FakeLLM({"conflicts": {"conflicts": []}}) + spec = _spec( + "update_employee", + "update_employee.a", + "update_employee.b", + "update_employee.c", + "update_employee.d", + ) + + await find_conflicts(llm, [spec]) + + assert llm.count("conflicts") == 3 + + +async def test_found_conflicts_are_returned_with_no_resolution(): + llm = FakeLLM({"conflicts": {"conflicts": [_raw_conflict()]}}) + spec = _spec("update_employee", "update_employee.a", "update_employee.b") + + conflicts = await find_conflicts(llm, [spec]) + + assert len(conflicts) == 1 + assert conflicts[0].kind == "scope" + assert conflicts[0].resolution is None + + +async def test_duplicate_conflicts_are_deduped_by_id(): + llm = FakeLLM({"conflicts": {"conflicts": [_raw_conflict(), _raw_conflict()]}}) + spec = _spec("update_employee", "update_employee.a", "update_employee.b") + + assert len(await find_conflicts(llm, [spec])) == 1 + + +async def test_a_malformed_conflict_entry_is_skipped(): + llm = FakeLLM({"conflicts": {"conflicts": [{"id": "x"}, _raw_conflict()]}}) + spec = _spec("update_employee", "update_employee.a", "update_employee.b") + + conflicts = await find_conflicts(llm, [spec]) + + assert [c.id for c in conflicts] == ["conflict.update_employee.scope"] + + +async def test_a_response_without_a_conflicts_key_is_tolerated(): + llm = FakeLLM({"conflicts": {"unexpected": True}}) + spec = _spec("update_employee", "update_employee.a", "update_employee.b") + + assert await find_conflicts(llm, [spec]) == [] + + +async def test_conflicts_referencing_unknown_items_are_dropped(): + # A conflict must point at real policy items to be actionable. + llm = FakeLLM({"conflicts": {"conflicts": [_raw_conflict(policies=("made.up",))]}}) + spec = _spec("update_employee", "update_employee.a", "update_employee.b") + + assert await find_conflicts(llm, [spec]) == [] + + +# --- attach_conflicts ------------------------------------------------------ + + +def _conflict(id_, policies) -> Conflict: + return Conflict( + id=id_, + name="c", + kind="scope", + conflicting_policies=list(policies), + description="d", + question="q", + ) + + +def test_a_single_tool_conflict_attaches_to_that_tool(): + specs = [ + _spec("update_employee", "update_employee.a"), + _spec("set_passport", "set_passport.a"), + ] + conflict = _conflict("c1", ["update_employee.a", "update_employee.b"]) + + attach_conflicts(specs, [conflict]) + + assert [c.id for c in specs[0].conflicts] == ["c1"] + assert specs[1].conflicts == [] + + +def test_a_cross_tool_conflict_attaches_to_every_involved_tool(): + specs = [ + _spec("update_employee", "update_employee.a"), + _spec("set_passport", "set_passport.a"), + _spec("get_visa", "get_visa.a"), + ] + conflict = _conflict("c1", ["update_employee.a", "set_passport.a"]) + + attach_conflicts(specs, [conflict]) + + assert [c.id for c in specs[0].conflicts] == ["c1"] + assert [c.id for c in specs[1].conflicts] == ["c1"] + assert specs[2].conflicts == [] + + +def test_attaching_replaces_rather_than_accumulates(): + # Rerunning detection must not double up conflicts already on the spec. + spec = _spec("update_employee", "update_employee.a") + conflict = _conflict("c1", ["update_employee.a"]) + + attach_conflicts([spec], [conflict]) + attach_conflicts([spec], [conflict]) + + assert len(spec.conflicts) == 1 + + +def test_a_conflict_naming_an_absent_tool_is_ignored(): + spec = _spec("update_employee", "update_employee.a") + conflict = _conflict("c1", ["nonexistent_tool.a"]) + + attach_conflicts([spec], [conflict]) + + assert spec.conflicts == [] + + +def test_attaching_nothing_clears_previous_conflicts(): + spec = _spec("update_employee", "update_employee.a") + spec.conflicts = [_conflict("stale", ["update_employee.a"])] + + attach_conflicts([spec], []) + + assert spec.conflicts == [] diff --git a/tests/buildtime/gen_spec_v2/test_context.py b/tests/buildtime/gen_spec_v2/test_context.py new file mode 100644 index 0000000..2d0d39f --- /dev/null +++ b/tests/buildtime/gen_spec_v2/test_context.py @@ -0,0 +1,66 @@ +"""The per-run context every stage renders its prompt slice from.""" + +from toolguard.buildtime.gen_spec.data_types import ToolInfo, ToolInfoParam +from toolguard.buildtime.gen_spec_v2.context import GenContext + +POLICY = """# Access Control + +## Data Access + +- **HR** may view and edit all employees' data. +- Only **HR** may add a new employee. +""" + + +def _tool(name: str, description: str) -> ToolInfo: + return ToolInfo( + name=name, + summary="", + description=description, + parameters={ + "user_id": ToolInfoParam( + type="int", description="the employee", required=True + ) + }, + signature=f"{name}(user_id: int) -> dict", + ) + + +def _ctx() -> GenContext: + return GenContext.build( + policy_text=POLICY, + tools=[ + _tool("get_employee", "Return an employee"), + _tool("add_employee", "Add"), + ], + system_vars={"department": ["HR", "Finance"]}, + ) + + +def test_policy_renders_verbatim_including_headings(): + # v2 does not split the document into bullets, so structure the LLM can + # use for context (headings, section order) must survive. + assert _ctx().render_policy() == POLICY + + +def test_tools_overview_lists_every_tool(): + rendered = _ctx().render_tools_overview() + + assert "- get_employee: Return an employee" in rendered + assert "- add_employee: Add" in rendered + + +def test_tool_detail_includes_parameters_and_signature(): + rendered = _ctx().render_tool_detail(_tool("get_employee", "Return an employee")) + + assert "get_employee" in rendered + assert "user_id" in rendered + assert "the employee" in rendered + + +def test_system_vars_render_through_the_context(): + assert "input.extensions.subject.department" in _ctx().render_system_vars() + + +def test_tool_names_are_exposed_for_validation(): + assert _ctx().tool_names() == ["get_employee", "add_employee"] diff --git a/tests/buildtime/gen_spec_v2/test_examples_only.py b/tests/buildtime/gen_spec_v2/test_examples_only.py new file mode 100644 index 0000000..880ac11 --- /dev/null +++ b/tests/buildtime/gen_spec_v2/test_examples_only.py @@ -0,0 +1,165 @@ +"""Regenerating examples for specs already on disk. + +The counterpart to v1's `generate_guard_examples`: examples are what test +generation works from, and rewording them should not cost a full spec run. +""" + +from typing import Any, Dict + + +from toolguard.buildtime.gen_spec_v2.models import Trigger +from toolguard.buildtime.gen_spec_v2.pipeline import ( + generate_guard_examples_v2, + generate_guard_specs_v2, +) +from toolguard.buildtime.gen_spec_v2.serialize import load_spec + +from .conftest import POLICY, SYSTEM_VARS, FakeLLM, make_tool + +HR_RULE = "**HR** may view and edit all employees' data." +TOOLS = [ + make_tool("update_employee", "Update an employee"), + make_tool("get_employee", "Read"), +] + +FIRST_PASS: Dict[str, Any] = { + "create": { + "tool_info": {"is_read_only": False, "user_enrichment": "writes"}, + "policy_items": [ + { + "slug": "hr_only", + "name": "HR only may edit", + "description": "only HR", + "references": [HR_RULE], + } + ], + }, + "expand": {"policy_items": []}, + "review": {"is_relevant": True, "can_be_validated": True, "reason": "yes"}, + "enrich": { + "trigger": "post_tool", + "requires": { + "system_vars": ["department"], + "tool_history": [{"tool": "get_employee", "params": {}}], + "message_history": None, + }, + "references": [HR_RULE], + "pending_for_user": [], + }, + "examples": { + "compliance_examples": ["first compliance"], + "violation_examples": ["first violation"], + }, +} + +SECOND_PASS: Dict[str, Any] = { + "examples": { + "compliance_examples": ["An HR user in Finance edits a record."], + "violation_examples": ["An Engineering user edits another record."], + } +} + + +async def _seed(tmp_path) -> None: + await generate_guard_specs_v2( + POLICY, TOOLS, FakeLLM(FIRST_PASS), tmp_path, system_vars=SYSTEM_VARS + ) + + +async def test_examples_are_replaced_on_disk(tmp_path): + await _seed(tmp_path) + llm = FakeLLM(SECOND_PASS) + + await generate_guard_examples_v2( + POLICY, TOOLS, llm, tmp_path, system_vars=SYSTEM_VARS + ) + + spec = load_spec(tmp_path / "update_employee.json") + assert spec.policy_items[0].compliance_examples == [ + "An HR user in Finance edits a record." + ] + assert spec.policy_items[0].violation_examples == [ + "An Engineering user edits another record." + ] + + +async def test_only_the_examples_stage_runs(tmp_path): + await _seed(tmp_path) + llm = FakeLLM(SECOND_PASS) + + await generate_guard_examples_v2(POLICY, TOOLS, llm, tmp_path) + + assert llm.count("examples") > 0 + assert llm.count("create") == 0 + assert llm.count("expand") == 0 + assert llm.count("review") == 0 + assert llm.count("enrich") == 0 + + +async def test_everything_else_about_the_item_is_preserved(tmp_path): + await _seed(tmp_path) + + await generate_guard_examples_v2(POLICY, TOOLS, FakeLLM(SECOND_PASS), tmp_path) + + item = load_spec(tmp_path / "update_employee.json").policy_items[0] + assert item.id == "update_employee.hr_only" + assert item.trigger == Trigger.post_tool + assert item.requires.system_vars == ["department"] + assert item.requires.tool_history is not None + assert item.references == [HR_RULE] + + +async def test_a_fixed_example_count_is_requested(tmp_path): + await _seed(tmp_path) + llm = FakeLLM(SECOND_PASS) + + await generate_guard_examples_v2(POLICY, TOOLS, llm, tmp_path, example_number=3) + + assert "exactly 3" in llm.calls_for("examples")[0]["content"] + + +async def test_specs_can_be_passed_instead_of_loaded(tmp_path): + specs = await generate_guard_specs_v2( + POLICY, TOOLS, FakeLLM(FIRST_PASS), tmp_path, system_vars=SYSTEM_VARS + ) + llm = FakeLLM(SECOND_PASS) + + result = await generate_guard_examples_v2(POLICY, TOOLS, llm, tmp_path, specs=specs) + + assert result[0].policy_items[0].compliance_examples == [ + "An HR user in Finance edits a record." + ] + + +async def test_a_spec_whose_tool_is_unknown_is_skipped(tmp_path): + await _seed(tmp_path) + llm = FakeLLM(SECOND_PASS) + + # Only get_employee is supplied, so update_employee's spec has no tool to + # render into the prompt and must be left alone rather than guessed at. + await generate_guard_examples_v2(POLICY, TOOLS[1:], llm, tmp_path) + + spec = load_spec(tmp_path / "update_employee.json") + assert spec.policy_items[0].compliance_examples == ["first compliance"] + + +async def test_an_empty_directory_is_not_an_error(tmp_path): + result = await generate_guard_examples_v2( + POLICY, TOOLS, FakeLLM(SECOND_PASS), tmp_path + ) + + assert result == [] + + +async def test_rewrite_false_leaves_the_file_untouched(tmp_path): + await _seed(tmp_path) + before = (tmp_path / "update_employee.json").read_text(encoding="utf-8") + + result = await generate_guard_examples_v2( + POLICY, TOOLS, FakeLLM(SECOND_PASS), tmp_path, rewrite=False + ) + + assert (tmp_path / "update_employee.json").read_text(encoding="utf-8") == before + assert result[0].policy_items[0].compliance_examples == [ + "An HR user in Finance edits a record." + ] diff --git a/tests/buildtime/gen_spec_v2/test_gen_py_contract.py b/tests/buildtime/gen_spec_v2/test_gen_py_contract.py new file mode 100644 index 0000000..eb9006b --- /dev/null +++ b/tests/buildtime/gen_spec_v2/test_gen_py_contract.py @@ -0,0 +1,156 @@ +"""The adapter's contract with `gen_py`, checked without calling an LLM. + +`generate_guards_code` needs a model to write guard bodies, but everything it +does to a spec *before* that is deterministic: it copies the spec, drops +skipped items, drops specs left empty, and derives python identifiers from +each item's name. Those steps are what an adapter change would break, so they +are exercised here directly against the real `gen_py` helpers. + +The full path — v2 -> adapter -> generated guards that compile and pass their +own tests — is `tests/buildtime/e2e/test_gen_spec_v2_codegen.py`, which needs +LLM credentials. +""" + +from pathlib import Path + +import pytest + +from toolguard.buildtime.gen_py.naming_conv import ( + guard_fn_name, + guard_item_fn_module_name, + guard_item_fn_name, +) +from toolguard.buildtime.gen_py.naming_conv import ( + # aliased: pytest would collect a module-level name starting with `test_` + test_fn_module_name as item_test_module_name, +) +from toolguard.buildtime.gen_spec_v2.adapter import spec_v2_to_v1, specs_v2_to_v1 +from toolguard.buildtime.gen_spec_v2.models import PolicyItemV2, SpecV2 +from toolguard.buildtime.gen_spec_v2.serialize import load_spec +from toolguard.runtime.data_types import ToolGuardSpec + +from .conftest import requires_corpus + +SPEC_DIR = Path("tests/data/specs_v2") +EMPLOYEE_SPECS = sorted(SPEC_DIR.glob("*.json")) + + +def _codegen_prepare(specs): + """Reproduce what `gen_toolguards.generate_*` does before any LLM call.""" + for spec in specs: + for item in spec.policy_items: + item.name = item.name.replace(".", "_") + + return [ + spec + for spec in [ + ToolGuardSpec( + tool_name=spec.tool_name, + policy_items=[i for i in spec.policy_items if not i.skip], + ) + for spec in specs + ] + if len(spec.policy_items) > 0 + ] + + +@requires_corpus +@pytest.mark.parametrize("path", EMPLOYEE_SPECS, ids=lambda p: p.stem) +def test_every_codegen_bound_item_yields_a_usable_python_identifier(path: Path): + # Only unskipped items are named by codegen. Skipped ones are not, which + # matters here: `to_snake_case` leaves punctuation alone, so a name + # containing parentheses would produce an invalid identifier — a + # pre-existing v1 limitation that the adapter must not walk into. + v1 = spec_v2_to_v1(load_spec(path)) + + assert guard_fn_name(v1).isidentifier() + for item in v1.policy_items: + if item.skip: + continue + assert guard_item_fn_name(item).isidentifier() + assert guard_item_fn_module_name(item).isidentifier() + + +def test_the_disambiguation_suffix_stays_a_valid_identifier(): + # The adapter's own suffix must not introduce punctuation that snake-casing + # would leave in place. + spec = SpecV2( + tool_name="t", + policy_items=[ + PolicyItemV2( + id="t.first", name="same rule", description="d", references=["r"] + ), + PolicyItemV2( + id="t.second", name="same rule", description="d", references=["r"] + ), + ], + ) + + for item in spec_v2_to_v1(spec).policy_items: + assert guard_item_fn_name(item).isidentifier() + + +@requires_corpus +@pytest.mark.parametrize("path", EMPLOYEE_SPECS, ids=lambda p: p.stem) +def test_items_of_one_tool_never_share_a_generated_file(path: Path): + # Two items mapping to one module name would overwrite each other's guard. + v1 = spec_v2_to_v1(load_spec(path)) + modules = [guard_item_fn_module_name(i) for i in v1.policy_items] + tests = [item_test_module_name(i) for i in v1.policy_items] + + assert len(set(modules)) == len(modules) + assert len(set(tests)) == len(tests) + + +def test_colliding_names_survive_codegens_dot_replacement(): + # gen_py rewrites '.' to '_' in item names in place, which could re-collide + # two names the adapter had just disambiguated. + spec = SpecV2( + tool_name="t", + policy_items=[ + PolicyItemV2(id="t.a", name="rule v1.0", description="d", references=["r"]), + PolicyItemV2(id="t.b", name="rule v1_0", description="d", references=["r"]), + ], + ) + + prepared = _codegen_prepare([spec_v2_to_v1(spec)]) + modules = [guard_item_fn_module_name(i) for i in prepared[0].policy_items] + + assert len(set(modules)) == 2 + + +@requires_corpus +def test_codegen_receives_only_enforceable_items(): + specs = [spec_v2_to_v1(load_spec(p)) for p in EMPLOYEE_SPECS] + + prepared = _codegen_prepare(specs) + + assert {s.tool_name for s in prepared} == { + "add_employee", + "create_time_off_request", + "set_passport", + "set_visa", + "update_employee", + "update_passport", + "update_visa", + } + assert sum(len(s.policy_items) for s in prepared) == 15 + + +@requires_corpus +def test_a_spec_with_nothing_enforceable_is_dropped_before_codegen(): + specs = [spec_v2_to_v1(load_spec(SPEC_DIR / "get_employee.json"))] + + assert _codegen_prepare(specs) == [] + + +@requires_corpus +def test_the_unattachable_global_spec_never_reaches_codegen(): + # A spec whose tool_name has no tool behind it would make codegen generate + # a guard for nothing. + all_specs = [load_spec(p) for p in EMPLOYEE_SPECS] + tools = [p.stem for p in EMPLOYEE_SPECS if p.stem != "global"] + + converted = specs_v2_to_v1(all_specs, known_tools=tools) + + assert "global" not in {s.tool_name for s in converted} diff --git a/tests/buildtime/gen_spec_v2/test_pipeline.py b/tests/buildtime/gen_spec_v2/test_pipeline.py new file mode 100644 index 0000000..6ad6823 --- /dev/null +++ b/tests/buildtime/gen_spec_v2/test_pipeline.py @@ -0,0 +1,310 @@ +"""The per-tool pipeline and the three public entry points.""" + +import json + +import pytest + +from toolguard.buildtime.gen_spec_v2.models import PendingType, Trigger +from toolguard.buildtime.gen_spec_v2.pipeline import ( + SpecV2Options, + generate_guard_specs_v2, + generate_guard_specs_v2_full, + generate_spec_conflicts_v2, +) +from toolguard.buildtime.gen_spec_v2.serialize import load_spec + +from .conftest import POLICY, SYSTEM_VARS, FakeLLM, make_tool + +HR_RULE = "**HR** may view and edit all employees' data." +SALARY_RULE = "When an employee's **salary** is set or updated, it must be a positive amount (greater than zero)." + +TOOLS = [ + make_tool("update_employee", "Update an employee"), + make_tool("get_employee", "Read"), +] + + +def _responses(**overrides): + base = { + "create": { + "tool_info": {"is_read_only": False, "user_enrichment": "writes"}, + "policy_items": [ + { + "slug": "hr_only", + "name": "HR only may edit", + "description": "only HR", + "references": [HR_RULE], + }, + { + "slug": "salary_positive", + "name": "Salary must be positive", + "description": "salary > 0", + "references": [SALARY_RULE], + }, + ], + }, + "expand": {"policy_items": []}, + "review": {"is_relevant": True, "can_be_validated": True, "reason": "yes"}, + "enrich": lambda content: ( + { + "trigger": "pre_tool", + "requires": { + "system_vars": ["department"] if "only HR" in content else [], + "tool_history": None, + "message_history": None, + }, + "references": [HR_RULE] if "only HR" in content else [SALARY_RULE], + "pending_for_user": [], + } + ), + "examples": { + "compliance_examples": ["An HR user edits a record."], + "violation_examples": ["An engineer edits another record."], + }, + "conflicts": {"conflicts": []}, + } + base.update(overrides) + return base + + +async def test_specs_are_written_one_file_per_tool(tmp_path): + llm = FakeLLM(_responses()) + + specs = await generate_guard_specs_v2( + POLICY, TOOLS, llm, tmp_path, system_vars=SYSTEM_VARS, source_doc="policy.md" + ) + + assert {s.tool_name for s in specs} == {"update_employee", "get_employee"} + assert (tmp_path / "update_employee.json").exists() + assert load_spec(tmp_path / "update_employee.json").source_doc == "policy.md" + + +async def test_every_stage_contributes_to_the_written_spec(tmp_path): + llm = FakeLLM(_responses()) + + await generate_guard_specs_v2( + POLICY, TOOLS[:1], llm, tmp_path, system_vars=SYSTEM_VARS + ) + spec = load_spec(tmp_path / "update_employee.json") + + hr = next(i for i in spec.policy_items if i.id == "update_employee.hr_only") + assert hr.trigger == Trigger.pre_tool + assert hr.requires.system_vars == ["department"] + assert hr.compliance_examples == ["An HR user edits a record."] + assert hr.references == [HR_RULE] + assert spec.debug.tool_info.user_enrichment == "writes" + + +async def test_references_are_grounded_against_the_policy_document(tmp_path): + # The model quotes the rule without its markdown; the written spec should + # carry the document's exact text. + responses = _responses() + responses["create"] = { + "policy_items": [ + { + "slug": "hr_only", + "name": "HR only", + "description": "only HR", + "references": ["HR may view and edit all employees' data."], + } + ] + } + responses["enrich"] = { + "trigger": "pre_tool", + "requires": {"system_vars": [], "tool_history": None, "message_history": None}, + "references": [], + "pending_for_user": [], + } + llm = FakeLLM(responses) + + await generate_guard_specs_v2(POLICY, TOOLS[:1], llm, tmp_path) + spec = load_spec(tmp_path / "update_employee.json") + + assert spec.policy_items[0].references == [HR_RULE] + + +async def test_tools2guard_limits_which_specs_are_generated(tmp_path): + llm = FakeLLM(_responses()) + + specs = await generate_guard_specs_v2( + POLICY, TOOLS, llm, tmp_path, tools2guard=["update_employee"] + ) + + assert [s.tool_name for s in specs] == ["update_employee"] + assert not (tmp_path / "get_employee.json").exists() + + +async def test_an_unknown_tool_in_tools2guard_is_rejected(tmp_path): + llm = FakeLLM(_responses()) + + with pytest.raises(ValueError, match="nope"): + await generate_guard_specs_v2( + POLICY, TOOLS, llm, tmp_path, tools2guard=["nope"] + ) + + +async def test_a_tool_with_no_items_still_gets_a_spec_file(tmp_path): + # An empty spec is a real answer — "no rule governs this tool" — and its + # absence would otherwise be indistinguishable from a failed run. + llm = FakeLLM(_responses(create={"policy_items": []})) + + await generate_guard_specs_v2(POLICY, TOOLS[:1], llm, tmp_path) + + assert (tmp_path / "update_employee.json").exists() + assert load_spec(tmp_path / "update_employee.json").policy_items == [] + + +async def test_pending_gaps_reach_the_written_spec(tmp_path): + responses = _responses() + responses["enrich"] = { + "trigger": "pre_tool", + "requires": {"system_vars": [], "tool_history": None, "message_history": None}, + "references": [], + "pending_for_user": [ + { + "type": "missing_variable", + "detail": "no blacklist variable exists", + "question": "where does blacklist status come from?", + "suggested_source": "system_vars:blacklist", + } + ], + } + llm = FakeLLM(responses) + + await generate_guard_specs_v2(POLICY, TOOLS[:1], llm, tmp_path) + spec = load_spec(tmp_path / "update_employee.json") + + pending = spec.policy_items[0].pending_for_user[0] + assert pending.type == PendingType.missing_var + assert pending.suggested_source == "system_vars:blacklist" + + +async def test_a_failing_tool_does_not_abort_the_others(tmp_path): + class Boom(FakeLLM): + async def chat_json(self, messages): + # Key on the tool under generation, not the catalog every prompt + # carries, so only get_employee's pipeline fails. + if "Tool name: get_employee" in messages[-1]["content"]: + raise RuntimeError("model exploded") + return await super().chat_json(messages) + + llm = Boom(_responses()) + + specs = await generate_guard_specs_v2(POLICY, TOOLS, llm, tmp_path) + + assert [s.tool_name for s in specs] == ["update_employee"] + + +async def test_on_tool_error_raise_propagates(tmp_path): + class Boom(FakeLLM): + async def chat_json(self, messages): + raise RuntimeError("model exploded") + + llm = Boom(_responses()) + + with pytest.raises(RuntimeError, match="model exploded"): + await generate_guard_specs_v2( + POLICY, + TOOLS, + llm, + tmp_path, + options=SpecV2Options(on_tool_error="raise"), + ) + + +async def test_examples_can_be_switched_off(tmp_path): + llm = FakeLLM(_responses()) + + await generate_guard_specs_v2( + POLICY, + TOOLS[:1], + llm, + tmp_path, + options=SpecV2Options(include_examples=False), + ) + + assert llm.count("examples") == 0 + + +async def test_system_vars_can_come_from_a_file(tmp_path): + path = tmp_path / "sys_var.json" + path.write_text(json.dumps(SYSTEM_VARS), encoding="utf-8") + llm = FakeLLM(_responses()) + + await generate_guard_specs_v2( + POLICY, TOOLS[:1], llm, tmp_path / "out", system_vars=path + ) + spec = load_spec(tmp_path / "out" / "update_employee.json") + + hr = next(i for i in spec.policy_items if i.id == "update_employee.hr_only") + assert hr.requires.system_vars == ["department"] + + +# --- conflicts entry point ------------------------------------------------- + + +async def test_conflicts_entry_point_loads_specs_from_disk(tmp_path): + llm = FakeLLM(_responses()) + await generate_guard_specs_v2(POLICY, TOOLS[:1], llm, tmp_path) + + conflict_llm = FakeLLM( + _responses( + conflicts={ + "conflicts": [ + { + "id": "conflict.update_employee.scope", + "name": "scope clash", + "kind": "scope", + "conflicting_policies": [ + "update_employee.hr_only", + "update_employee.salary_positive", + ], + "description": "both apply", + "question": "which wins?", + } + ] + } + ) + ) + + specs = await generate_spec_conflicts_v2(conflict_llm, tmp_path) + + assert [c.id for c in specs[0].conflicts] == ["conflict.update_employee.scope"] + assert load_spec(tmp_path / "update_employee.json").conflicts + + +async def test_conflicts_entry_point_leaves_other_specs_untouched(tmp_path): + llm = FakeLLM(_responses()) + await generate_guard_specs_v2(POLICY, TOOLS, llm, tmp_path) + before = (tmp_path / "get_employee.json").read_text(encoding="utf-8") + + await generate_spec_conflicts_v2(FakeLLM(_responses()), tmp_path) + + assert (tmp_path / "get_employee.json").read_text(encoding="utf-8") == before + + +async def test_regenerating_one_tool_leaves_the_other_file_alone(tmp_path): + llm = FakeLLM(_responses()) + await generate_guard_specs_v2(POLICY, TOOLS, llm, tmp_path) + before = (tmp_path / "get_employee.json").read_text(encoding="utf-8") + + await generate_guard_specs_v2( + POLICY, TOOLS, FakeLLM(_responses()), tmp_path, tools2guard=["update_employee"] + ) + + assert (tmp_path / "get_employee.json").read_text(encoding="utf-8") == before + + +# --- all-in-one ------------------------------------------------------------ + + +async def test_full_run_generates_specs_then_conflicts(tmp_path): + llm = FakeLLM(_responses()) + + specs = await generate_guard_specs_v2_full( + POLICY, TOOLS, llm, tmp_path, system_vars=SYSTEM_VARS + ) + + assert {s.tool_name for s in specs} == {"update_employee", "get_employee"} + assert llm.count("create") == 2 + assert llm.count("conflicts") > 0 diff --git a/tests/buildtime/gen_spec_v2/test_prompts.py b/tests/buildtime/gen_spec_v2/test_prompts.py new file mode 100644 index 0000000..4fc00af --- /dev/null +++ b/tests/buildtime/gen_spec_v2/test_prompts.py @@ -0,0 +1,112 @@ +"""Prompt assembly: every stage must see the inputs it is judging against. + +In v1 the feasibility reviewer was never shown the system variables it was +implicitly judging enforceability against. These tests pin the inputs each +stage receives so that cannot silently regress. +""" + +import pytest + +from toolguard.buildtime.gen_spec.data_types import ToolInfo, ToolInfoParam +from toolguard.buildtime.gen_spec_v2 import prompts +from toolguard.buildtime.gen_spec_v2.context import GenContext + +POLICY = "- **HR** may view and edit all employees' data.\n" + +TOOL = ToolInfo( + name="update_employee", + summary="", + description="Update an employee", + parameters={"user_id": ToolInfoParam(type="int", description="who", required=True)}, + signature="update_employee(user_id: int) -> dict", +) + +OTHER = ToolInfo( + name="get_employee", + summary="", + description="Read an employee", + parameters={}, + signature="get_employee(user_id: int) -> dict", +) + +ITEM = { + "id": "update_employee.hr_only", + "name": "HR only", + "description": "Only HR may edit", + "references": ["**HR** may view and edit all employees' data."], +} + + +def _ctx() -> GenContext: + return GenContext.build(POLICY, [TOOL, OTHER], {"department": ["HR", "Finance"]}) + + +@pytest.mark.parametrize( + "name", + ["create", "expand", "review", "enrich", "examples", "conflicts"], +) +def test_every_system_prompt_loads_and_demands_json(name): + system = prompts.system(name) + + assert system.strip() + assert "JSON" in system + + +def test_json_only_suffix_is_appended(): + assert "first character" in prompts.system("create").lower() + + +def test_create_user_content_carries_policy_tool_and_system_vars(): + content = prompts.create_user(_ctx(), TOOL) + + assert POLICY.strip() in content + assert "update_employee(user_id: int)" in content + assert "input.extensions.subject.department" in content + assert "- get_employee: Read an employee" in content + + +def test_expand_user_content_lists_existing_items(): + content = prompts.expand_user(_ctx(), TOOL, [ITEM]) + + assert "update_employee.hr_only" in content + assert POLICY.strip() in content + + +def test_review_user_content_shows_system_vars_and_other_tools(): + content = prompts.review_user(_ctx(), TOOL, ITEM) + + # v1's reviewer judged "can this be validated" without ever seeing these. + assert "input.extensions.subject.department" in content + assert "get_employee" in content + + +def test_enrich_user_content_carries_the_policy_for_reference_validation(): + content = prompts.enrich_user(_ctx(), TOOL, ITEM) + + assert POLICY.strip() in content + assert "input.extensions.subject.department" in content + assert "get_employee" in content + + +def test_enrich_user_content_excludes_the_tool_being_enriched_from_other_tools(): + content = prompts.enrich_user(_ctx(), TOOL, ITEM) + others_block = content.split("Other tools")[1] + + assert "get_employee" in others_block + assert "- update_employee:" not in others_block + + +def test_examples_user_content_carries_system_var_values(): + content = prompts.examples_user(_ctx(), TOOL, ITEM) + + assert "Finance" in content + assert "Only HR may edit" in content + + +def test_conflicts_user_content_names_the_tool_and_both_sides(): + other_item = {**ITEM, "id": "update_employee.self_service"} + content = prompts.conflicts_user("update_employee", ITEM, [other_item]) + + assert "update_employee" in content + assert "update_employee.hr_only" in content + assert "update_employee.self_service" in content diff --git a/tests/buildtime/gen_spec_v2/test_reconcile.py b/tests/buildtime/gen_spec_v2/test_reconcile.py new file mode 100644 index 0000000..a0d11e2 --- /dev/null +++ b/tests/buildtime/gen_spec_v2/test_reconcile.py @@ -0,0 +1,201 @@ +"""Combining several enrich votes into one answer. + +The enrich stage asks the model the same question N times and reconciles the +answers here. The rules are deliberately asymmetric: a missing requirement +is worse than a spurious one, so `system_vars` unions while +`message_history` needs a majority. +""" + +from toolguard.buildtime.gen_spec_v2.models import PendingType, Trigger +from toolguard.buildtime.gen_spec_v2.reconcile import reconcile + + +def vote( + trigger="pre_tool", + system_vars=None, + tool_history=None, + message_history=None, + references=None, + pending=None, +): + return { + "trigger": trigger, + "requires": { + "system_vars": system_vars or [], + "tool_history": tool_history, + "message_history": message_history, + }, + "references": references or [], + "pending_for_user": pending or [], + } + + +# --- trigger --------------------------------------------------------------- + + +def test_majority_trigger_wins(): + result = reconcile([vote(trigger="post_tool")] * 2 + [vote()]) + + assert result.trigger == Trigger.post_tool + + +def test_tied_trigger_resolves_to_pre_tool(): + # A pre-tool guard that should have been post-tool blocks a call; the + # reverse lets one through. On a tie, prefer blocking. + result = reconcile([vote(), vote(trigger="post_tool")]) + + assert result.trigger == Trigger.pre_tool + + +def test_no_votes_yields_pre_tool_and_empty_requirements(): + result = reconcile([]) + + assert result.trigger == Trigger.pre_tool + assert result.requires.system_vars == [] + assert result.requires.tool_history is None + assert result.requires.message_history is None + + +# --- system_vars ----------------------------------------------------------- + + +def test_system_vars_are_the_sorted_union_of_every_vote(): + result = reconcile( + [vote(system_vars=["user_id"]), vote(system_vars=["department", "user_id"])] + ) + + assert result.requires.system_vars == ["department", "user_id"] + + +# --- tool_history ---------------------------------------------------------- + + +def test_tool_history_is_none_when_a_strict_majority_say_none(): + history = [ + {"tool": "get_employee", "params": {"user_id": "input.arguments.user_id"}} + ] + result = reconcile([vote(), vote(), vote(tool_history=history)]) + + assert result.requires.tool_history is None + + +def test_longest_tool_history_wins_when_the_majority_want_one(): + short = [{"tool": "get_employee", "params": {}}] + long = [ + {"tool": "get_employee", "params": {}}, + {"tool": "get_leave_balance", "params": {"year": "2026"}}, + ] + result = reconcile([vote(tool_history=short), vote(tool_history=long), vote()]) + + assert result.requires.tool_history is not None + assert [e.tool for e in result.requires.tool_history] == [ + "get_employee", + "get_leave_balance", + ] + + +def test_tool_history_params_survive_reconciliation(): + history = [ + {"tool": "get_employee", "params": {"user_id": "input.arguments.user_id"}} + ] + result = reconcile([vote(tool_history=history), vote(tool_history=history)]) + + assert result.requires.tool_history is not None + assert result.requires.tool_history[0].params == { + "user_id": "input.arguments.user_id" + } + + +# --- message_history ------------------------------------------------------- + + +def test_message_history_needs_a_strict_majority(): + assert ( + reconcile([vote(message_history=True), vote(), vote()]).requires.message_history + is None + ) + + +def test_message_history_is_true_on_a_majority(): + result = reconcile([vote(message_history=True)] * 2 + [vote()]) + + assert result.requires.message_history is True + + +def test_message_history_is_none_rather_than_false_when_unneeded(): + # The on-disk format uses null, not false, for "not needed". + assert reconcile([vote(), vote()]).requires.message_history is None + + +# --- references ------------------------------------------------------------ + + +def test_references_union_in_first_seen_order(): + result = reconcile([vote(references=["b", "a"]), vote(references=["a", "c"])]) + + assert result.references == ["b", "a", "c"] + + +# --- pending_for_user ------------------------------------------------------ + + +def _pending(type_="missing_tool", question="which tool?", detail="d"): + return {"type": type_, "detail": detail, "question": question} + + +def test_pending_entries_dedupe_by_type_and_question(): + result = reconcile([vote(pending=[_pending()]), vote(pending=[_pending()])]) + + assert len(result.pending_for_user) == 1 + assert result.pending_for_user[0].type == PendingType.missing_tool + + +def test_pending_entries_differing_in_question_are_both_kept(): + result = reconcile( + [ + vote(pending=[_pending(question="which tool?")]), + vote(pending=[_pending(question="add one?")]), + ] + ) + + assert len(result.pending_for_user) == 2 + + +def test_pending_type_alias_is_normalized(): + result = reconcile([vote(pending=[_pending(type_="missing_variable")])]) + + assert result.pending_for_user[0].type == PendingType.missing_var + + +def test_pending_entry_with_an_unknown_type_is_dropped(): + result = reconcile([vote(pending=[_pending(type_="vibes")])]) + + assert result.pending_for_user == [] + + +def test_pending_entry_missing_a_question_is_dropped(): + result = reconcile([vote(pending=[{"type": "missing_tool", "detail": "d"}])]) + + assert result.pending_for_user == [] + + +# --- malformed votes ------------------------------------------------------- + + +def test_a_vote_missing_every_key_is_tolerated(): + result = reconcile([{}, vote(system_vars=["user_id"])]) + + assert result.requires.system_vars == ["user_id"] + assert result.trigger == Trigger.pre_tool + + +def test_a_non_dict_vote_is_ignored(): + result = reconcile(["nonsense", vote(trigger="post_tool")]) + + assert result.trigger == Trigger.post_tool + + +def test_a_vote_with_a_malformed_requires_is_tolerated(): + result = reconcile([{"trigger": "pre_tool", "requires": None}, vote()]) + + assert result.requires.system_vars == [] diff --git a/tests/buildtime/gen_spec_v2/test_refmatch.py b/tests/buildtime/gen_spec_v2/test_refmatch.py new file mode 100644 index 0000000..4e36d03 --- /dev/null +++ b/tests/buildtime/gen_spec_v2/test_refmatch.py @@ -0,0 +1,249 @@ +"""Grounding a quoted reference back onto the policy document. + +Each test here corresponds to a way v1's `find_mismatched_references` fails: +lowercase-only normalization, offset projection that assumes normalization +preserves length, a two-part split that accepts unrelated fragments, and no +fuzzy matching at all. +""" + +from toolguard.buildtime.gen_spec_v2.models import PolicyItemV2, SpecV2 +from toolguard.buildtime.gen_spec_v2.refmatch import ( + ground, + ground_spec, + normalize, + segments, +) + +POLICY = """# Employee Hub Policy + +## Data Access + +- An employee may view and edit only their **own data** — home address, passport, visa, emergency contact, and bank account. +- **HR** may view and edit all employees' data. +- An employee's `salary` may be updated only by **HR** or by that employee's **direct manager**. + +## Confirmation + +- A user may be deleted only after the requester provides explicit confirmation in exactly this form: + `I request to delete user [USER NAME] with the following user id: [USER ID]`. + +The agent must be careful. It should never guess an identity. Ambiguity is resolved by asking. +""" + +HR_RULE = "**HR** may view and edit all employees' data." + + +# --- normalize ------------------------------------------------------------- + + +def test_normalize_maps_every_output_char_back_to_its_source(): + text = "A **B**\nC" + norm, offsets = normalize(text) + + assert len(offsets) == len(norm) + # Every offset points at the character it came from. + for i, char in enumerate(norm): + if char != " ": + assert text[offsets[i]].lower() == char + + +def test_normalize_collapses_whitespace_and_drops_emphasis(): + norm, _ = normalize("**HR** may\n edit") + + assert norm == "hr may edit" + + +def test_normalize_unifies_dashes(): + assert normalize("a — b")[0] == normalize("a - b")[0] + + +# --- segments -------------------------------------------------------------- + + +def test_bullet_segments_exclude_the_list_marker(): + texts = [s.text for s in segments(POLICY)] + + assert HR_RULE in texts + + +def test_prose_is_split_into_sentences(): + texts = [s.text for s in segments(POLICY)] + + assert "It should never guess an identity." in texts + + +def test_segment_spans_point_at_the_original_text(): + for segment in segments(POLICY): + assert POLICY[segment.start : segment.end] == segment.text + + +# --- ground ---------------------------------------------------------------- + + +def test_exact_quote_grounds_to_itself(): + assert ground(HR_RULE, POLICY) == [HR_RULE] + + +def test_line_wrapped_quote_grounds(): + wrapped = "**HR** may view and\n edit all employees' data." + + assert ground(wrapped, POLICY) == [HR_RULE] + + +def test_quote_with_markdown_stripped_grounds_to_the_marked_up_original(): + assert ground("HR may view and edit all employees' data.", POLICY) == [HR_RULE] + + +def test_quote_with_a_hyphen_grounds_to_the_em_dash_original(): + quote = ( + "An employee may view and edit only their **own data** - home address, " + "passport, visa, emergency contact, and bank account." + ) + + grounded = ground(quote, POLICY) + + assert len(grounded) == 1 + assert "—" in grounded[0] + + +def test_mid_sentence_fragment_snaps_out_to_the_whole_rule(): + # v1 would return the bare fragment; a reference should be a whole rule. + assert ground("may view and edit all employees' data", POLICY) == [HR_RULE] + + +def test_paraphrase_grounds_to_the_closest_rule(): + quote = "An employee's salary may only be updated by HR or by the employee's direct manager." + + grounded = ground(quote, POLICY) + + assert len(grounded) == 1 + assert grounded[0].startswith("An employee's `salary` may be updated only by") + + +def test_quote_spanning_two_segments_returns_both(): + quote = ( + "A user may be deleted only after the requester provides explicit " + "confirmation in exactly this form: `I request to delete user [USER NAME] " + "with the following user id: [USER ID]`." + ) + + grounded = ground(quote, POLICY) + + assert len(grounded) == 2 + assert grounded[0].endswith("in exactly this form:") + assert grounded[1].startswith("`I request to delete user") + + +def test_unrelated_quote_does_not_ground(): + assert ground("Passengers may check two bags on domestic flights.", POLICY) == [] + + +def test_two_unrelated_fragments_are_not_stitched_together(): + # v1's fallback accepted any two halves found anywhere in the document. + assert ground("employees' data bank account salary manager", POLICY) == [] + + +# --- ground_spec ----------------------------------------------------------- + + +def _spec(references) -> SpecV2: + return SpecV2( + tool_name="update_employee", + policy_items=[ + PolicyItemV2( + id="update_employee.x", + name="x", + description="d", + references=references, + ) + ], + ) + + +def test_ground_spec_rewrites_references_to_verbatim_spans(): + spec = _spec(["HR may view and edit all employees' data."]) + + ungrounded = ground_spec(spec, POLICY) + + assert spec.policy_items[0].references == [HR_RULE] + assert ungrounded == [] + assert spec.debug.notes is None + + +def test_ground_spec_dedupes_references_that_ground_to_one_rule(): + spec = _spec([HR_RULE, "may view and edit all employees' data"]) + + ground_spec(spec, POLICY) + + assert spec.policy_items[0].references == [HR_RULE] + + +def test_ground_spec_keeps_an_ungrounded_reference_and_records_it(): + invented = "Employees must wear a badge at all times." + spec = _spec([HR_RULE, invented]) + + ungrounded = ground_spec(spec, POLICY) + + assert spec.policy_items[0].references == [HR_RULE, invented] + assert ungrounded == [invented] + assert spec.debug.notes is not None + assert invented in str(spec.debug.notes) + + +def test_ground_spec_leaves_an_item_without_references_alone(): + spec = _spec([]) + + assert ground_spec(spec, POLICY) == [] + assert spec.policy_items[0].references == [] + + +def test_an_item_grounded_in_nothing_is_archived(): + # An item whose every quote is absent from the policy document is not a rule + # from this policy at all — in practice it has quoted a tool's own docstring. + # Keeping it would let a rule nobody wrote reach the spec. + invented = "Only transfer if the user explicitly asks for a human agent." + spec = _spec([invented]) + + ungrounded = ground_spec(spec, POLICY) + + assert spec.policy_items == [] + assert ungrounded == [invented] + entry = spec.debug.archive[0] + assert entry["id"] == "update_employee.x" + assert entry["stage"] == "refmatch" + assert invented in entry["references"] + assert "policy document" in entry["reason"] + + +def test_an_item_with_one_grounded_quote_survives(): + spec = _spec(["Employees must wear a badge.", HR_RULE]) + + ground_spec(spec, POLICY) + + assert len(spec.policy_items) == 1 + assert spec.debug.archive == [] + + +def test_only_the_ungrounded_item_is_archived(): + spec = SpecV2( + tool_name="update_employee", + policy_items=[ + PolicyItemV2( + id="update_employee.good", + name="g", + description="d", + references=[HR_RULE], + ), + PolicyItemV2( + id="update_employee.bad", + name="b", + description="d", + references=["Badges are mandatory."], + ), + ], + ) + + ground_spec(spec, POLICY) + + assert [i.id for i in spec.policy_items] == ["update_employee.good"] + assert [a["id"] for a in spec.debug.archive] == ["update_employee.bad"] diff --git a/tests/buildtime/gen_spec_v2/test_refmatch_real_policy.py b/tests/buildtime/gen_spec_v2/test_refmatch_real_policy.py new file mode 100644 index 0000000..d2e36a7 --- /dev/null +++ b/tests/buildtime/gen_spec_v2/test_refmatch_real_policy.py @@ -0,0 +1,48 @@ +"""Grounding against the real employee policy document. + +Every reference in the ground-truth specs was quoted from +``tests/data/specs_v2_inputs/guidance.txt``, so grounding each one must +return that same span. This is the regression gate on the matcher: a +normalization or segmentation change that starts mangling real references +fails here, where the synthetic tests would not notice. +""" + +from pathlib import Path + +import pytest + +from toolguard.buildtime.gen_spec_v2.refmatch import ground +from toolguard.buildtime.gen_spec_v2.serialize import load_spec + +from .conftest import requires_corpus + +GUIDANCE = Path("tests/data/specs_v2_inputs/guidance.txt") +SPEC_DIR = Path("tests/data/specs_v2") + +pytestmark = requires_corpus + + +def _references(): + doc = GUIDANCE.read_text(encoding="utf-8") + for path in sorted(SPEC_DIR.glob("*.json")): + for item in load_spec(path).policy_items: + for reference in item.references: + yield doc, item.id, reference + + +# Guarded: this module reads the corpus at import time, and an unguarded read +# aborts collection for the whole suite when tests/data/ is absent. +ALL_REFERENCES = list(_references()) if GUIDANCE.is_file() else [] + + +def test_the_corpus_is_not_empty(): + assert len(ALL_REFERENCES) == 95 + + +@pytest.mark.parametrize( + "doc,item_id,reference", + ALL_REFERENCES, + ids=[f"{item_id}#{i}" for i, (_, item_id, _) in enumerate(ALL_REFERENCES)], +) +def test_real_reference_grounds_to_itself(doc, item_id, reference): + assert ground(reference, doc) == [reference] diff --git a/tests/buildtime/gen_spec_v2/test_serialize.py b/tests/buildtime/gen_spec_v2/test_serialize.py new file mode 100644 index 0000000..c9fe91c --- /dev/null +++ b/tests/buildtime/gen_spec_v2/test_serialize.py @@ -0,0 +1,50 @@ +"""Byte parity of the v2 serializer against smith's ground-truth specs. + +These fixtures were produced by smith's generator and are the definition of +the on-disk format: key order and omit-when-empty behavior included. A +load -> dump round trip must reproduce them exactly, so a future change to +the models cannot quietly reorder or drop a key. +""" + +from pathlib import Path + +import pytest + +from toolguard.buildtime.gen_spec_v2.serialize import dump_spec_str, load_spec + +from .conftest import requires_corpus + +FIXTURE_DIR = Path("tests/data/specs_v2") +FIXTURES = sorted(FIXTURE_DIR.glob("*.json")) + +pytestmark = requires_corpus + + +def _expected(path: Path) -> str: + """The fixture's text with the one deliberate normalization applied. + + Smith's enrich prompt asks for `missing_var`, but two fixtures contain + `missing_variable` (LLM drift that smith's archive check never matched). + v2 normalizes the alias, so those two files legitimately round-trip to + the canonical spelling. + """ + return path.read_text(encoding="utf-8").replace( + '"type": "missing_variable"', '"type": "missing_var"' + ) + + +def test_fixtures_are_present(): + assert len(FIXTURES) == 28 + + +@pytest.mark.parametrize("path", FIXTURES, ids=lambda p: p.stem) +def test_fixture_round_trips_byte_identical(path: Path): + assert dump_spec_str(load_spec(path)) == _expected(path) + + +def test_pending_type_alias_is_normalized(): + spec = load_spec(FIXTURE_DIR / "set_passport.json") + pending = [p for item in spec.policy_items for p in item.pending_for_user] + types = {p.type.value for p in pending} + assert "missing_var" in types + assert "missing_variable" not in types diff --git a/tests/buildtime/gen_spec_v2/test_stages.py b/tests/buildtime/gen_spec_v2/test_stages.py new file mode 100644 index 0000000..cec3a7d --- /dev/null +++ b/tests/buildtime/gen_spec_v2/test_stages.py @@ -0,0 +1,432 @@ +"""The five per-tool generation stages, driven by a scripted LLM. + +Every stage reads LLM output, so the tests concentrate on two things: the +transformation the stage is responsible for, and what it does with a response +that is valid JSON but not the shape it asked for. +""" + +from toolguard.buildtime.gen_spec_v2.models import ( + PendingType, + PolicyItemV2, + Trigger, +) +from toolguard.buildtime.gen_spec_v2.stages import ( + run_create, + run_enrich, + run_examples, + run_expand, + run_review, +) + +from .conftest import FakeLLM + +HR_RULE = "**HR** may view and edit all employees' data." +SALARY_RULE = "When an employee's **salary** is set or updated, it must be a positive amount (greater than zero)." + + +def _raw(slug="hr_only", name="HR only", references=None, description="only HR may"): + return { + "slug": slug, + "name": name, + "description": description, + "references": [HR_RULE] if references is None else references, + } + + +def _item(id_="update_employee.hr_only", name="HR only", **kwargs) -> PolicyItemV2: + return PolicyItemV2( + id=id_, + name=name, + description=kwargs.pop("description", "only HR may"), + references=kwargs.pop("references", [HR_RULE]), + **kwargs, + ) + + +# --- create ---------------------------------------------------------------- + + +async def test_create_builds_items_with_tool_prefixed_ids(ctx, tool): + llm = FakeLLM( + { + "create": { + "tool_info": {"is_read_only": False, "user_enrichment": "writes"}, + "policy_items": [_raw()], + } + } + ) + + tool_info, items = await run_create(llm, ctx, tool) + + assert tool_info.user_enrichment == "writes" + assert [i.id for i in items] == ["update_employee.hr_only"] + assert items[0].trigger == Trigger.pre_tool + + +async def test_create_never_trusts_an_llm_supplied_tool_prefix(ctx, tool): + # Ids are assigned in code, so a model naming the wrong tool cannot produce + # an id that points at a different tool's spec. + raw = _raw() + raw.pop("slug") + llm = FakeLLM({"create": {"policy_items": [{**raw, "id": "wrong_tool.whatever"}]}}) + + _, items = await run_create(llm, ctx, tool) + + assert items[0].id == "update_employee.whatever" + + +async def test_create_prefers_the_slug_over_a_supplied_id(ctx, tool): + llm = FakeLLM( + {"create": {"policy_items": [{**_raw(), "id": "wrong_tool.whatever"}]}} + ) + + _, items = await run_create(llm, ctx, tool) + + assert items[0].id == "update_employee.hr_only" + + +async def test_create_drops_an_item_with_no_references(ctx, tool): + # An item quoting nothing is not grounded in the policy document. + llm = FakeLLM({"create": {"policy_items": [_raw(references=[])]}}) + + _, items = await run_create(llm, ctx, tool) + + assert items == [] + + +async def test_create_drops_an_unnamed_item(ctx, tool): + llm = FakeLLM({"create": {"policy_items": [_raw(name="")]}}) + + _, items = await run_create(llm, ctx, tool) + + assert items == [] + + +async def test_create_disambiguates_colliding_slugs(ctx, tool): + llm = FakeLLM({"create": {"policy_items": [_raw(), _raw(name="HR only again")]}}) + + _, items = await run_create(llm, ctx, tool) + + assert [i.id for i in items] == [ + "update_employee.hr_only", + "update_employee.hr_only_2", + ] + + +async def test_create_defaults_a_missing_tool_info(ctx, tool): + llm = FakeLLM({"create": {"policy_items": []}}) + + tool_info, items = await run_create(llm, ctx, tool) + + assert tool_info.is_read_only is False + assert items == [] + + +async def test_create_tolerates_a_response_without_policy_items(ctx, tool): + llm = FakeLLM({"create": {"nonsense": True}}) + + _, items = await run_create(llm, ctx, tool) + + assert items == [] + + +# --- expand ---------------------------------------------------------------- + + +async def test_expand_appends_new_items(ctx, tool): + llm = FakeLLM( + { + "expand": { + "policy_items": [_raw(slug="salary_positive", name="Salary positive")] + } + } + ) + + items = await run_expand(llm, ctx, tool, [_item()], iterations=1) + + assert [i.id for i in items] == [ + "update_employee.hr_only", + "update_employee.salary_positive", + ] + + +async def test_expand_stops_early_when_a_pass_adds_nothing(ctx, tool): + llm = FakeLLM({"expand": {"policy_items": []}}) + + await run_expand(llm, ctx, tool, [_item()], iterations=3) + + assert llm.count("expand") == 1 + + +async def test_expand_runs_every_iteration_while_it_keeps_finding_items(ctx, tool): + counter = {"n": 0} + + def respond(_content): + counter["n"] += 1 + return { + "policy_items": [ + _raw(slug=f"extra_{counter['n']}", name=f"E{counter['n']}") + ] + } + + llm = FakeLLM({"expand": respond}) + + items = await run_expand(llm, ctx, tool, [_item()], iterations=3) + + assert llm.count("expand") == 3 + assert len(items) == 4 + + +async def test_expand_does_not_duplicate_an_existing_rule(ctx, tool): + llm = FakeLLM({"expand": {"policy_items": [_raw()]}}) + + items = await run_expand(llm, ctx, tool, [_item()], iterations=1) + + assert len(items) == 1 + + +async def test_expand_drops_an_ungrounded_item(ctx, tool): + llm = FakeLLM({"expand": {"policy_items": [_raw(slug="new", references=[])]}}) + + items = await run_expand(llm, ctx, tool, [_item()], iterations=1) + + assert len(items) == 1 + + +# --- review ---------------------------------------------------------------- + + +def _vote(relevant=True, valid=True, reason="because"): + return {"is_relevant": relevant, "can_be_validated": valid, "reason": reason} + + +async def test_review_keeps_an_item_the_majority_approves(ctx, tool): + llm = FakeLLM({"review": _vote()}) + + kept, archived = await run_review(llm, ctx, tool, [_item()], votes=3) + + assert len(kept) == 1 + assert archived == [] + assert llm.count("review") == 3 + + +async def test_review_archives_an_item_the_majority_rejects(ctx, tool): + llm = FakeLLM({"review": _vote(relevant=False, reason="not this tool")}) + + kept, archived = await run_review(llm, ctx, tool, [_item()], votes=3) + + assert kept == [] + assert archived[0]["id"] == "update_employee.hr_only" + assert archived[0]["stage"] == "review" + assert "not this tool" in archived[0]["reason"] + + +async def test_review_archives_an_item_that_cannot_be_validated(ctx, tool): + llm = FakeLLM({"review": _vote(valid=False)}) + + kept, _ = await run_review(llm, ctx, tool, [_item()], votes=3) + + assert kept == [] + + +async def test_review_archive_keeps_the_references_for_traceability(ctx, tool): + llm = FakeLLM({"review": _vote(relevant=False)}) + + _, archived = await run_review(llm, ctx, tool, [_item()], votes=1) + + assert archived[0]["references"] == [HR_RULE] + + +async def test_review_treats_a_malformed_vote_as_a_rejection(ctx, tool): + llm = FakeLLM({"review": {"reason": "no verdict"}}) + + kept, _ = await run_review(llm, ctx, tool, [_item()], votes=1) + + assert kept == [] + + +async def test_review_makes_no_calls_without_items(ctx, tool): + llm = FakeLLM({"review": _vote()}) + + kept, archived = await run_review(llm, ctx, tool, [], votes=3) + + assert (kept, archived) == ([], []) + assert llm.count("review") == 0 + + +# --- enrich ---------------------------------------------------------------- + + +def _enrich_vote( + trigger="pre_tool", + system_vars=None, + tool_history=None, + message_history=None, + references=None, + pending=None, +): + return { + "trigger": trigger, + "requires": { + "system_vars": system_vars if system_vars is not None else [], + "tool_history": tool_history, + "message_history": message_history, + }, + "references": [HR_RULE] if references is None else references, + "pending_for_user": pending or [], + } + + +async def test_enrich_writes_trigger_and_requires(ctx, tool): + llm = FakeLLM( + {"enrich": _enrich_vote(trigger="post_tool", system_vars=["department"])} + ) + + items = await run_enrich(llm, ctx, tool, [_item()], votes=3) + + assert items[0].trigger == Trigger.post_tool + assert items[0].requires.system_vars == ["department"] + + +async def test_enrich_drops_an_undeclared_system_variable(ctx, tool): + llm = FakeLLM({"enrich": _enrich_vote(system_vars=["department", "is_admin"])}) + + items = await run_enrich(llm, ctx, tool, [_item()], votes=1) + + assert items[0].requires.system_vars == ["department"] + + +async def test_enrich_drops_a_tool_history_entry_for_an_unknown_tool(ctx, tool): + llm = FakeLLM( + { + "enrich": _enrich_vote( + tool_history=[ + {"tool": "get_employee", "params": {}}, + {"tool": "check_blacklist", "params": {}}, + ] + ) + } + ) + + items = await run_enrich(llm, ctx, tool, [_item()], votes=1) + + history = items[0].requires.tool_history + assert history is not None + assert [e.tool for e in history] == ["get_employee"] + + +async def test_enrich_only_validates_references_never_adds_them(ctx, tool): + invented = "Employees must wear a badge." + llm = FakeLLM({"enrich": _enrich_vote(references=[HR_RULE, invented])}) + + items = await run_enrich(llm, ctx, tool, [_item()], votes=1) + + assert items[0].references == [HR_RULE] + + +async def test_enrich_keeps_original_references_when_validation_blanks_them(ctx, tool): + llm = FakeLLM({"enrich": _enrich_vote(references=[])}) + + items = await run_enrich(llm, ctx, tool, [_item()], votes=1) + + assert items[0].references == [HR_RULE] + + +async def test_enrich_records_a_pending_gap(ctx, tool): + llm = FakeLLM( + { + "enrich": _enrich_vote( + pending=[ + { + "type": "missing_var", + "detail": "no blacklist variable", + "question": "where does blacklist status come from?", + } + ] + ) + } + ) + + items = await run_enrich(llm, ctx, tool, [_item()], votes=1) + + assert items[0].pending_for_user[0].type == PendingType.missing_var + + +async def test_enrich_keeps_an_item_that_is_blocked_and_otherwise_unenforceable( + ctx, tool +): + # Smith archives these. v2 keeps them: surfacing the gap is the point, and + # the adapter marks anything with pending_for_user as not codegen-able. + llm = FakeLLM( + { + "enrich": _enrich_vote( + pending=[ + { + "type": "missing_tool", + "detail": "no email tool", + "question": "which tool sends the email?", + } + ] + ) + } + ) + + items = await run_enrich(llm, ctx, tool, [_item()], votes=1) + + assert len(items) == 1 + assert items[0].pending_for_user[0].type == PendingType.missing_tool + + +async def test_enrich_tolerates_an_empty_response(ctx, tool): + llm = FakeLLM({"enrich": {}}) + + items = await run_enrich(llm, ctx, tool, [_item()], votes=1) + + assert items[0].trigger == Trigger.pre_tool + assert items[0].references == [HR_RULE] + + +# --- examples -------------------------------------------------------------- + + +async def test_examples_are_written_onto_the_item(ctx, tool): + llm = FakeLLM( + { + "examples": { + "compliance_examples": ["An HR user edits a record."], + "violation_examples": ["An engineer edits someone else's record."], + } + } + ) + item = _item() + + await run_examples(llm, ctx, tool, [item]) + + assert item.compliance_examples == ["An HR user edits a record."] + assert item.violation_examples == ["An engineer edits someone else's record."] + + +async def test_examples_degrade_to_empty_on_a_misshaped_response(ctx, tool): + llm = FakeLLM({"examples": {"compliance_examples": "not a list"}}) + item = _item() + + await run_examples(llm, ctx, tool, [item]) + + assert item.compliance_examples == [] + assert item.violation_examples == [] + + +async def test_examples_can_request_a_fixed_count(ctx, tool): + llm = FakeLLM({"examples": {"compliance_examples": [], "violation_examples": []}}) + + await run_examples(llm, ctx, tool, [_item()], example_number=2) + + assert "exactly 2" in llm.calls_for("examples")[0]["content"] + + +async def test_examples_makes_no_calls_without_items(ctx, tool): + llm = FakeLLM({"examples": {}}) + + await run_examples(llm, ctx, tool, []) + + assert llm.count("examples") == 0 diff --git a/tests/buildtime/gen_spec_v2/test_sysvars.py b/tests/buildtime/gen_spec_v2/test_sysvars.py new file mode 100644 index 0000000..a3d9715 --- /dev/null +++ b/tests/buildtime/gen_spec_v2/test_sysvars.py @@ -0,0 +1,127 @@ +"""System variables: loading from a dict or a file, and prompt rendering. + +The rendered block is what stops the LLM inventing subject variables, so it +must name every declared variable, its access path, and its domain. +""" + +import json + +import pytest + +from .conftest import requires_corpus +from toolguard.buildtime.gen_spec_v2.sysvars import ( + keep_declared, + load_system_vars, + render_system_vars, +) + +EMPLOYEE_VARS = { + "user_name": "Bob", + "user_id": 1, + "department": ["Corporate Leadership", "Engineering", "Product", "HR", "Finance"], + "organization": ["IBM Corporation", "Red Hat", "Kyndryl"], +} + + +def test_dict_input_keeps_every_key_in_order(): + sv = load_system_vars(EMPLOYEE_VARS) + assert sv.names == ["user_name", "user_id", "department", "organization"] + + +def test_none_input_yields_no_variables(): + assert load_system_vars(None).names == [] + + +def test_path_input_reads_the_file(tmp_path): + path = tmp_path / "sys_var.json" + path.write_text(json.dumps(EMPLOYEE_VARS), encoding="utf-8") + + assert load_system_vars(path).names == list(EMPLOYEE_VARS) + assert load_system_vars(str(path)).raw["user_name"] == "Bob" + + +def test_non_object_json_is_rejected(tmp_path): + path = tmp_path / "sys_var.json" + path.write_text("[1, 2]", encoding="utf-8") + + with pytest.raises(ValueError): + load_system_vars(path) + + +def test_list_value_renders_as_allowed_values(): + rendered = render_system_vars(load_system_vars(EMPLOYEE_VARS)) + + assert ( + "- organization (input.extensions.subject.organization): allowed values = " + '["IBM Corporation", "Red Hat", "Kyndryl"]' in rendered + ) + + +def test_scalar_value_renders_as_an_example(): + rendered = render_system_vars(load_system_vars(EMPLOYEE_VARS)) + + assert "- user_id (input.extensions.subject.user_id): example value = 1" in rendered + + +def test_rendering_without_variables_says_so(): + rendered = render_system_vars(load_system_vars(None)) + + assert "no system variables" in rendered.lower() + + +def test_the_agents_own_action_catalog_is_ignored(): + # Real sys_var files carry the agent's action catalog alongside the acting + # user's attributes. Those two keys describe the tools, not the subject, and + # `action_description` is a 35-entry mapping that would be pure prompt noise. + sv = load_system_vars( + { + "user_id": 1, + "action_list": ["add_employee", "get_employee"], + "action_description": {"add_employee": "Create a record"}, + } + ) + + assert sv.names == ["user_id"] + assert "action_description" not in render_system_vars(sv) + assert "action_list" not in render_system_vars(sv) + + +def test_a_nested_mapping_is_still_a_subject_variable(): + # sys_var.json is not assumed to be flat: a structured attribute of the + # acting user is kept and rendered. + sv = load_system_vars({"entitlements": {"payroll": "read", "pii": "none"}}) + + assert sv.names == ["entitlements"] + assert '"payroll": "read"' in render_system_vars(sv) + + +def test_a_nested_list_is_still_a_subject_variable(): + sv = load_system_vars({"managed_teams": [{"id": 3, "name": "Platform"}]}) + + assert sv.names == ["managed_teams"] + assert "Platform" in render_system_vars(sv) + + +@requires_corpus +def test_the_real_employee_sys_var_file_yields_only_subject_variables(): + sv = load_system_vars("tests/data/specs_v2_inputs/system_vars.json") + + assert sv.names == ["user_name", "user_id", "department", "organization"] + + +def test_keep_declared_drops_invented_names(): + sv = load_system_vars(EMPLOYEE_VARS) + + assert keep_declared(["department", "is_admin", "user_id"], sv) == [ + "department", + "user_id", + ] + + +def test_keep_declared_dedupes_and_preserves_order(): + sv = load_system_vars(EMPLOYEE_VARS) + + assert keep_declared(["user_id", "department", "user_id"], sv) == [ + "user_id", + "department", + ] diff --git a/tests/buildtime/gen_spec_v2/test_tools_input.py b/tests/buildtime/gen_spec_v2/test_tools_input.py new file mode 100644 index 0000000..721d6b7 --- /dev/null +++ b/tests/buildtime/gen_spec_v2/test_tools_input.py @@ -0,0 +1,64 @@ +"""Accepting every tool-description shape v2 supports. + +v1's ``_tools_to_tool_infos`` documents a ``list[ToolInfo]`` branch but +raises on it. v2 needs that branch, because a caller holding MCP tool +definitions already has exactly that. +""" + +from pathlib import Path + +import pytest + +from toolguard.buildtime.gen_spec.data_types import ToolInfo, ToolInfoParam +from toolguard.buildtime.gen_spec_v2.tools_input import to_tool_infos +from toolguard.buildtime.utils.open_api import OpenAPI + +OAS_PATH = Path("tests/examples/appointments/appointments_oas.json") + + +def _tool_info(name: str) -> ToolInfo: + return ToolInfo( + name=name, + summary="", + description=f"{name} does a thing", + parameters={ + "user_id": ToolInfoParam(type="int", description=None, required=True) + }, + signature="", + ) + + +def test_tool_info_list_passes_through(): + infos = [_tool_info("get_employee"), _tool_info("set_passport")] + + assert [t.name for t in to_tool_infos(infos)] == ["get_employee", "set_passport"] + + +def test_openapi_dict_is_converted(): + oas = OpenAPI.load_from(OAS_PATH) + + names = [t.name for t in to_tool_infos(oas.model_dump(by_alias=True))] + + assert "add_payment_method" in names + + +def test_callables_are_converted(): + def transfer(user_id: int, amount: float) -> bool: + """Transfer money.""" + return True + + infos = to_tool_infos([transfer]) + + assert infos[0].name == "transfer" + assert set(infos[0].parameters) == {"user_id", "amount"} + + +def test_unsupported_input_is_rejected(): + # Deliberately off-type: the guard exists for callers without type checking. + with pytest.raises(NotImplementedError): + to_tool_infos([object()]) # type: ignore[list-item] + + +def test_empty_list_is_rejected(): + with pytest.raises(ValueError): + to_tool_infos([]) From 730ded03274f5fdd69a783a5f55537d3cf40d2ec Mon Sep 17 00:00:00 2001 From: naamaz Date: Tue, 11 Aug 2026 13:59:05 +0300 Subject: [PATCH 2/6] docs: note the ground-truth corpus is not committed, and the counts without it Removing tests/data/ from the commit changes what a fresh clone measures: 273 passed + 13 skipped rather than 488 passed. Record both, and how to restore the corpus. Co-Authored-By: Claude Opus 5 (1M context) --- .../2026-08-11-gen-spec-v2-test-baseline.md | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/superpowers/evidence/2026-08-11-gen-spec-v2-test-baseline.md b/docs/superpowers/evidence/2026-08-11-gen-spec-v2-test-baseline.md index 1d00dca..bb00b29 100644 --- a/docs/superpowers/evidence/2026-08-11-gen-spec-v2-test-baseline.md +++ b/docs/superpowers/evidence/2026-08-11-gen-spec-v2-test-baseline.md @@ -1,7 +1,7 @@ # gen_spec_v2 — test baseline Date: 2026-08-11 -Base commit: `e31b21c` (version 0.2.21), branch `sys_var`, nothing committed +Base commit: `e31b21c` (version 0.2.21); the work itself landed as `da24902` on branch `sys_var` LLM for e2e runs: `claude-sonnet-4-6` via azure Design: [2026-08-10-gen-spec-v2-design.md](../specs/2026-08-10-gen-spec-v2-design.md) @@ -10,6 +10,27 @@ inferred; §7 records where a result came from an earlier state of the tree. ## How to reproduce +### The corpus is not committed + +The 488-passed figure below needs the employee ground-truth corpus, which is deliberately +not in the repository. Without it 13 tests skip and the count is **273 passed, 13 skipped** +— a fresh clone runs green either way, it just proves less. To get the full number, copy +from smith: + +```bash +mkdir -p tests/data/specs_v2 tests/data/specs_v2_inputs +SMITH=../smith/examples/employee/smith +cp "$SMITH"/smith_outputs/ground_truth_specs/*.json tests/data/specs_v2/ +cp "$SMITH"/{guidance.txt,system_vars.json,tool_definitions.json} tests/data/specs_v2_inputs/ +``` + +Five test files read it — `test_serialize.py`, `test_refmatch_real_policy.py`, +`test_adapter.py`, `test_gen_py_contract.py`, `test_sysvars.py` — via the +`requires_corpus` mark in `tests/buildtime/gen_spec_v2/conftest.py`. No shipped code +under `src/` touches it. + +### Commands + ```bash # No LLM needed. Definitive, ~53s. PYTHONPATH=tests python -m pytest tests -q --ignore=tests/tmp --ignore=tests/buildtime/e2e @@ -28,6 +49,7 @@ python -m pyright src/toolguard/buildtime/gen_spec_v2 tests/buildtime/gen_spec_v | Suite | Result | Time | |---|---|---| | Non-e2e (unit + contract) | **488 passed**, 0 failed, 3 pre-existing warnings | 52.9s | +| Non-e2e without the ground-truth corpus | **273 passed, 13 skipped** | 21.4s | | v2 e2e — calculator, 4 tool-input shapes | **4 passed** | in the 417s below | | v2 e2e — tau2 `simple` | **passed** | " | | v2 e2e — tau2 `complex_api` | **failed** — transient LLM invalid-JSON, §4 | " | From f0333ebeae65e528b9afc3e5390459f421a1bf4e Mon Sep 17 00:00:00 2001 From: naamaz Date: Tue, 11 Aug 2026 14:28:04 +0300 Subject: [PATCH 3/6] Commit a six-spec employee slice so the corpus tests run in a plain clone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 13 corpus tests skipped on a fresh checkout, which meant the byte-parity, reference-grounding and codegen-identifier gates were not running for anyone who had not copied smith's employee output into tests/data/ by hand. tests/examples/employee_mini/ is that corpus at minimum size: six of the 28 ground-truth specs, copied byte for byte, chosen so between them they cover 70 of the 71 distinct key paths in the full set — both sides of the adapter's skip decision, all three pending types, both triggers, tool_history, message_history, conflicts present and absent, and the empty spec. Real generator output rather than hand-written fixtures, so these gates keep testing what a model actually produces. Three derivations, documented in the directory's README: the six-spec subset, guidance.txt truncated to the rules those specs quote (seven unreferenced rule bullets dropped), and global.json's conflict targets pruned to the items the subset contains. The five test files now read this slice unconditionally, so requires_corpus is gone and tests/data/ is gitignored — the full 28-spec example and its benchmarks belong to the evaluate-toolguard project. Verified with tests/data/ absent: 332 passed, 0 skipped, 0 failed. The gates run fewer cases than the full corpus (27 references rather than 95, 6 fixtures rather than 28) but gate the same behaviors; both counts are in the baseline doc. Co-Authored-By: Claude Opus 5 (1M context) --- .gitignore | 3 + .../2026-08-11-gen-spec-v2-test-baseline.md | 93 +++---- .../specs/2026-08-10-gen-spec-v2-design.md | 7 +- tests/buildtime/gen_spec_v2/conftest.py | 33 +-- tests/buildtime/gen_spec_v2/test_adapter.py | 6 +- .../gen_spec_v2/test_gen_py_contract.py | 15 +- .../gen_spec_v2/test_refmatch_real_policy.py | 26 +- tests/buildtime/gen_spec_v2/test_serialize.py | 10 +- tests/buildtime/gen_spec_v2/test_sysvars.py | 6 +- tests/examples/employee_mini/README.md | 57 +++++ tests/examples/employee_mini/guidance.txt | 70 ++++++ .../specs/create_time_off_request.json | 157 ++++++++++++ .../employee_mini/specs/get_employee.json | 79 ++++++ .../examples/employee_mini/specs/global.json | 114 +++++++++ .../employee_mini/specs/list_employees.json | 64 +++++ .../employee_mini/specs/set_passport.json | 153 ++++++++++++ .../employee_mini/specs/update_employee.json | 230 ++++++++++++++++++ tests/examples/employee_mini/system_vars.json | 78 ++++++ 18 files changed, 1096 insertions(+), 105 deletions(-) create mode 100644 tests/examples/employee_mini/README.md create mode 100644 tests/examples/employee_mini/guidance.txt create mode 100644 tests/examples/employee_mini/specs/create_time_off_request.json create mode 100644 tests/examples/employee_mini/specs/get_employee.json create mode 100644 tests/examples/employee_mini/specs/global.json create mode 100644 tests/examples/employee_mini/specs/list_employees.json create mode 100644 tests/examples/employee_mini/specs/set_passport.json create mode 100644 tests/examples/employee_mini/specs/update_employee.json create mode 100644 tests/examples/employee_mini/system_vars.json diff --git a/.gitignore b/.gitignore index ce930bf..a87b20f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ dist/toolguard-0.1.6-py3-none-any.whl dist/toolguard-0.1.6.tar.gz tmp .history/* +# The full employee ground truth, if copied in from evaluate-toolguard. The +# committed slice the tests actually use is tests/examples/employee_mini/. +tests/data/ diff --git a/docs/superpowers/evidence/2026-08-11-gen-spec-v2-test-baseline.md b/docs/superpowers/evidence/2026-08-11-gen-spec-v2-test-baseline.md index bb00b29..66d2bed 100644 --- a/docs/superpowers/evidence/2026-08-11-gen-spec-v2-test-baseline.md +++ b/docs/superpowers/evidence/2026-08-11-gen-spec-v2-test-baseline.md @@ -10,24 +10,18 @@ inferred; §7 records where a result came from an earlier state of the tree. ## How to reproduce -### The corpus is not committed +### Nothing to copy in — the fixtures are committed -The 488-passed figure below needs the employee ground-truth corpus, which is deliberately -not in the repository. Without it 13 tests skip and the count is **273 passed, 13 skipped** -— a fresh clone runs green either way, it just proves less. To get the full number, copy -from smith: +Everything below runs from a plain `git clone`. The five tests that need real +generator output read `tests/examples/employee_mini/` — a committed six-spec slice of +smith's employee output, with `guidance.txt` truncated to the rules those specs quote. +No shipped code under `src/` reads any fixture. -```bash -mkdir -p tests/data/specs_v2 tests/data/specs_v2_inputs -SMITH=../smith/examples/employee/smith -cp "$SMITH"/smith_outputs/ground_truth_specs/*.json tests/data/specs_v2/ -cp "$SMITH"/{guidance.txt,system_vars.json,tool_definitions.json} tests/data/specs_v2_inputs/ -``` - -Five test files read it — `test_serialize.py`, `test_refmatch_real_policy.py`, -`test_adapter.py`, `test_gen_py_contract.py`, `test_sysvars.py` — via the -`requires_corpus` mark in `tests/buildtime/gen_spec_v2/conftest.py`. No shipped code -under `src/` touches it. +The §1 numbers were first measured against the full 28-spec corpus at +`tests/data/specs_v2/`, which is now gitignored and lives in the evaluate-toolguard +project. Both counts are recorded below, with the older one marked as no longer +reproducible from this repo: what changed is the size of the parametrized sets, not +which behaviors are gated. ### Commands @@ -48,8 +42,8 @@ python -m pyright src/toolguard/buildtime/gen_spec_v2 tests/buildtime/gen_spec_v | Suite | Result | Time | |---|---|---| -| Non-e2e (unit + contract) | **488 passed**, 0 failed, 3 pre-existing warnings | 52.9s | -| Non-e2e without the ground-truth corpus | **273 passed, 13 skipped** | 21.4s | +| Non-e2e (unit + contract), committed fixtures | **332 passed**, 0 failed, 0 skipped, 3 pre-existing warnings | 72.2s | +| Non-e2e against the full 28-spec corpus (earlier in session; see §1) | **488 passed**, 0 failed | 52.9s | | v2 e2e — calculator, 4 tool-input shapes | **4 passed** | in the 417s below | | v2 e2e — tau2 `simple` | **passed** | " | | v2 e2e — tau2 `complex_api` | **failed** — transient LLM invalid-JSON, §4 | " | @@ -59,31 +53,37 @@ python -m pyright src/toolguard/buildtime/gen_spec_v2 tests/buildtime/gen_spec_v | pyright — `gen_spec_v2` + all new tests | **0 errors** | | | pyright — whole `src/toolguard` | 15 errors, **all pre-existing** in v1 modules | | -Total: **493 automated checks passing, 1 failing**, the failure being a transport-level -model error rather than a logic defect. - -## 1. Non-e2e tests — 488 passed - -No LLM. This is the suite to trust for regressions. - -| File | Tests | Covers | -|---|---:|---| -| `test_refmatch_real_policy.py` | 96 | all 95 real ground-truth references ground back to themselves | -| `test_gen_py_contract.py` | 61 | python identifiers, generated-file collisions, what codegen receives | -| `test_adapter.py` | 41 | `skip` truth table, debug preservation, whole employee corpus | -| `test_stages.py` | 31 | create/expand/review/enrich/examples, incl. misshaped LLM responses | -| `test_serialize.py` | 30 | byte parity against all 28 ground-truth fixtures | -| `test_refmatch.py` | 22 | grounding: exact, wrapped, markdown-stripped, dash, snap, multi-segment, archive-when-ungrounded | -| `test_reconcile.py` | 19 | vote reconciliation, malformed votes | -| `test_pipeline.py` | 15 | 3 entry points, per-tool isolation, partial regeneration | -| `test_prompts.py` | 14 | every stage sees the inputs it judges against | -| `test_sysvars.py` | 13 | dict/path loading, `action_list`/`action_description` exclusion, nested values kept | -| `test_conflicts.py` | 12 | detection bounds, routing to each involved tool | -| `test_examples_only.py` | 8 | `generate_guard_examples_v2` touches only the examples | -| `test_context.py` | 5 | prompt-slice rendering | -| `test_tools_input.py` | 5 | callables / OpenAPI dict / `list[ToolInfo]` | -| **v2 subtotal** | **372** | | -| pre-existing suite | 116 | unchanged by this work | +Total: **337 automated checks passing, 1 failing** on committed fixtures alone, the +failure being a transport-level model error rather than a logic defect. + +## 1. Non-e2e tests — 332 passed + +No LLM. This is the suite to trust for regressions. The `mini` column is what a plain +clone runs — the reproducible number. The `full` column is what these same tests +measured earlier in the session, when they read the 28-spec corpus at +`tests/data/specs_v2/`; reproducing it now means repointing `CORPUS_DIR` in +`tests/buildtime/gen_spec_v2/conftest.py`, since nothing reads that path any more. Only +the four corpus-parametrized files differ between the columns, and only in how many +cases they run — not in which behaviors they gate. + +| File | mini | full | Covers | +|---|---:|---:|---| +| `test_stages.py` | 31 | 31 | create/expand/review/enrich/examples, incl. misshaped LLM responses | +| `test_refmatch_real_policy.py` | 28 | 96 | every real reference grounds back to itself (27 vs 95) | +| `test_refmatch.py` | 22 | 22 | grounding: exact, wrapped, markdown-stripped, dash, snap, multi-segment, archive-when-ungrounded | +| `test_reconcile.py` | 19 | 19 | vote reconciliation, malformed votes | +| `test_adapter.py` | 19 | 41 | `skip` truth table, debug preservation, the employee corpus | +| `test_gen_py_contract.py` | 17 | 61 | python identifiers, generated-file collisions, what codegen receives | +| `test_pipeline.py` | 15 | 15 | 3 entry points, per-tool isolation, partial regeneration | +| `test_prompts.py` | 14 | 14 | every stage sees the inputs it judges against | +| `test_sysvars.py` | 13 | 13 | dict/path loading, `action_list`/`action_description` exclusion, nested values kept | +| `test_conflicts.py` | 12 | 12 | detection bounds, routing to each involved tool | +| `test_serialize.py` | 8 | 30 | byte parity against every ground-truth fixture (6 vs 28) | +| `test_examples_only.py` | 8 | 8 | `generate_guard_examples_v2` touches only the examples | +| `test_context.py` | 5 | 5 | prompt-slice rendering | +| `test_tools_input.py` | 5 | 5 | callables / OpenAPI dict / `list[ToolInfo]` | +| **v2 subtotal** | **216** | **372** | | +| pre-existing suite | 116 | 116 | unchanged by this work | The 3 warnings are pre-existing: one `PytestCollectionWarning` for a test class with `__init__`, two litellm coroutine warnings. @@ -118,6 +118,10 @@ needs only a `tool_history` lookup. Everything identity-, conversation-, result- question-dependent is skipped. That is the honest count of what today's runtime can enforce, and each `skip` condition disappears as the runtime gains that capability. +The committed six-spec slice reproduces the same ratio in miniature: **7 of 22 items +across 3 of 6 tools**, asserted in `test_gen_py_contract.py`. Re-measure the 28-spec +figure in evaluate-toolguard; the mini figure is the one that regresses in CI. + ## 4. The one failure, and the variance behind it `test_tau2_v2.test_tau2_complex_api` failed at `test_tau2_v2.py:99` — the "one spec per @@ -217,3 +221,6 @@ Report is written to `tests/tmp/e2e/guard_set_delta/guard_set_delta.md` on each - Prompt files are `lru_cache`d per process, so a prompt edited while a run is in flight does not take effect mid-run. - e2e results depend on the model. Record `MODEL_NAME` alongside any future comparison. +- §1's per-file counts were measured twice: the `full` column against the 28-spec corpus + in place, the `mini` column after it was replaced by + `tests/examples/employee_mini/`. The `mini` column is the one CI will reproduce. diff --git a/docs/superpowers/specs/2026-08-10-gen-spec-v2-design.md b/docs/superpowers/specs/2026-08-10-gen-spec-v2-design.md index 1220222..7eee59c 100644 --- a/docs/superpowers/specs/2026-08-10-gen-spec-v2-design.md +++ b/docs/superpowers/specs/2026-08-10-gen-spec-v2-design.md @@ -310,8 +310,11 @@ would generate wrong-enforcement guards. Documented, not fixed: point v2 at its | e2e | v2 → adapter → `generate_guards_code`: guards compile and their generated tests pass (calculator) | yes | | e2e | Employee policy + MCP server → shape parity with ground truth: every tool file present, ids well-formed, trigger/requires on every item, pending types in the enum | yes | -Ground-truth fixtures are copied into `tests/data/specs_v2/` to avoid a cross-repo test -dependency. +Ground-truth fixtures live in `tests/examples/employee_mini/` — a committed six-spec +slice of smith's employee output, chosen so the byte-parity, grounding and +identifier gates run in a plain clone without a cross-repo dependency. The full +28-spec example and its benchmarks belong to the evaluate-toolguard project; see +that directory's README for the three derivations applied. ## Implementation notes diff --git a/tests/buildtime/gen_spec_v2/conftest.py b/tests/buildtime/gen_spec_v2/conftest.py index 80993f4..d74728f 100644 --- a/tests/buildtime/gen_spec_v2/conftest.py +++ b/tests/buildtime/gen_spec_v2/conftest.py @@ -1,4 +1,4 @@ -"""A scripted LLM for the stage and pipeline tests. +"""A scripted LLM for the stage and pipeline tests, plus the corpus paths. Responses are keyed by the ``[STAGE:x]`` marker every v2 user prompt carries, so a test declares what each stage answers without caring about call order or @@ -15,27 +15,20 @@ from toolguard.buildtime.gen_spec_v2.context import GenContext from toolguard.buildtime.llm import I_TG_LLM -CORPUS_DIR = Path("tests/data/specs_v2") -"""Ground-truth employee specs. Not committed; see the test-baseline doc.""" - -CORPUS_INPUTS_DIR = Path("tests/data/specs_v2_inputs") -"""The policy document, tool definitions and system vars those specs came from.""" - -requires_corpus = pytest.mark.skipif( - not CORPUS_DIR.is_dir() or not CORPUS_INPUTS_DIR.is_dir(), - reason=( - "needs the employee ground-truth corpus under tests/data/ — copy " - "smith/examples/employee/smith/{smith_outputs/ground_truth_specs,guidance.txt," - "system_vars.json,tool_definitions.json} into tests/data/specs_v2{,_inputs}/" - ), -) -"""Skip a test that reads the ground-truth corpus, when it is not present. - -Every test that needs the corpus carries this, and every module that reads it at -import time guards that read, so a checkout without `tests/data/` still collects -and runs the rest of the suite instead of failing collection outright. +CORPUS_DIR = Path("tests/examples/employee_mini") +"""A committed six-spec slice of the employee ground truth. + +Real generator output, so the format, grounding and identifier gates test what +a model actually produces rather than what a synthetic fixture remembers to +include. The full 28-spec example and its benchmarks live in the +evaluate-toolguard project; see this directory's README for what was kept and +why. """ +CORPUS_SPECS_DIR = CORPUS_DIR / "specs" +CORPUS_GUIDANCE = CORPUS_DIR / "guidance.txt" +CORPUS_SYSTEM_VARS = CORPUS_DIR / "system_vars.json" + STAGE_MARKER = re.compile(r"\[STAGE:(\w+)\]") Response = Union[Dict[str, Any], Callable[[str], Dict[str, Any]], Exception] diff --git a/tests/buildtime/gen_spec_v2/test_adapter.py b/tests/buildtime/gen_spec_v2/test_adapter.py index e48fa21..af77255 100644 --- a/tests/buildtime/gen_spec_v2/test_adapter.py +++ b/tests/buildtime/gen_spec_v2/test_adapter.py @@ -26,9 +26,9 @@ from toolguard.buildtime.gen_spec_v2.serialize import load_spec from toolguard.runtime.data_types import ToolGuardSpec -from .conftest import requires_corpus +from .conftest import CORPUS_SPECS_DIR -SPEC_DIR = Path("tests/data/specs_v2") +SPEC_DIR = CORPUS_SPECS_DIR def _item(name="a rule", **kwargs) -> PolicyItemV2: @@ -152,7 +152,6 @@ def test_spec_level_fields_are_preserved_in_debug(): # --- the real corpus ------------------------------------------------------- -@requires_corpus @pytest.mark.parametrize("path", sorted(SPEC_DIR.glob("*.json")), ids=lambda p: p.stem) def test_every_ground_truth_spec_converts_and_validates(path: Path): v1 = spec_v2_to_v1(load_spec(path)) @@ -161,7 +160,6 @@ def test_every_ground_truth_spec_converts_and_validates(path: Path): assert ToolGuardSpec.model_validate(v1.model_dump()).tool_name == v1.tool_name -@requires_corpus def test_employee_corpus_skips_every_identity_dependent_rule(): v1 = spec_v2_to_v1(load_spec(SPEC_DIR / "update_employee.json")) unskipped = {i.name for i in v1.policy_items if not i.skip} diff --git a/tests/buildtime/gen_spec_v2/test_gen_py_contract.py b/tests/buildtime/gen_spec_v2/test_gen_py_contract.py index eb9006b..de45434 100644 --- a/tests/buildtime/gen_spec_v2/test_gen_py_contract.py +++ b/tests/buildtime/gen_spec_v2/test_gen_py_contract.py @@ -29,9 +29,9 @@ from toolguard.buildtime.gen_spec_v2.serialize import load_spec from toolguard.runtime.data_types import ToolGuardSpec -from .conftest import requires_corpus +from .conftest import CORPUS_SPECS_DIR -SPEC_DIR = Path("tests/data/specs_v2") +SPEC_DIR = CORPUS_SPECS_DIR EMPLOYEE_SPECS = sorted(SPEC_DIR.glob("*.json")) @@ -54,7 +54,6 @@ def _codegen_prepare(specs): ] -@requires_corpus @pytest.mark.parametrize("path", EMPLOYEE_SPECS, ids=lambda p: p.stem) def test_every_codegen_bound_item_yields_a_usable_python_identifier(path: Path): # Only unskipped items are named by codegen. Skipped ones are not, which @@ -90,7 +89,6 @@ def test_the_disambiguation_suffix_stays_a_valid_identifier(): assert guard_item_fn_name(item).isidentifier() -@requires_corpus @pytest.mark.parametrize("path", EMPLOYEE_SPECS, ids=lambda p: p.stem) def test_items_of_one_tool_never_share_a_generated_file(path: Path): # Two items mapping to one module name would overwrite each other's guard. @@ -119,32 +117,25 @@ def test_colliding_names_survive_codegens_dot_replacement(): assert len(set(modules)) == 2 -@requires_corpus def test_codegen_receives_only_enforceable_items(): specs = [spec_v2_to_v1(load_spec(p)) for p in EMPLOYEE_SPECS] prepared = _codegen_prepare(specs) assert {s.tool_name for s in prepared} == { - "add_employee", "create_time_off_request", "set_passport", - "set_visa", "update_employee", - "update_passport", - "update_visa", } - assert sum(len(s.policy_items) for s in prepared) == 15 + assert sum(len(s.policy_items) for s in prepared) == 7 -@requires_corpus def test_a_spec_with_nothing_enforceable_is_dropped_before_codegen(): specs = [spec_v2_to_v1(load_spec(SPEC_DIR / "get_employee.json"))] assert _codegen_prepare(specs) == [] -@requires_corpus def test_the_unattachable_global_spec_never_reaches_codegen(): # A spec whose tool_name has no tool behind it would make codegen generate # a guard for nothing. diff --git a/tests/buildtime/gen_spec_v2/test_refmatch_real_policy.py b/tests/buildtime/gen_spec_v2/test_refmatch_real_policy.py index d2e36a7..d7a7559 100644 --- a/tests/buildtime/gen_spec_v2/test_refmatch_real_policy.py +++ b/tests/buildtime/gen_spec_v2/test_refmatch_real_policy.py @@ -1,25 +1,21 @@ """Grounding against the real employee policy document. -Every reference in the ground-truth specs was quoted from -``tests/data/specs_v2_inputs/guidance.txt``, so grounding each one must -return that same span. This is the regression gate on the matcher: a -normalization or segmentation change that starts mangling real references -fails here, where the synthetic tests would not notice. +Every reference in the ground-truth specs was quoted by a model out of +``employee_mini/guidance.txt``, so grounding each one must return that same +span. This is the regression gate on the matcher: a normalization or +segmentation change that starts mangling real references fails here, where the +synthetic tests would not notice. """ -from pathlib import Path - import pytest from toolguard.buildtime.gen_spec_v2.refmatch import ground from toolguard.buildtime.gen_spec_v2.serialize import load_spec -from .conftest import requires_corpus - -GUIDANCE = Path("tests/data/specs_v2_inputs/guidance.txt") -SPEC_DIR = Path("tests/data/specs_v2") +from .conftest import CORPUS_GUIDANCE, CORPUS_SPECS_DIR -pytestmark = requires_corpus +GUIDANCE = CORPUS_GUIDANCE +SPEC_DIR = CORPUS_SPECS_DIR def _references(): @@ -30,13 +26,11 @@ def _references(): yield doc, item.id, reference -# Guarded: this module reads the corpus at import time, and an unguarded read -# aborts collection for the whole suite when tests/data/ is absent. -ALL_REFERENCES = list(_references()) if GUIDANCE.is_file() else [] +ALL_REFERENCES = list(_references()) def test_the_corpus_is_not_empty(): - assert len(ALL_REFERENCES) == 95 + assert len(ALL_REFERENCES) == 27 @pytest.mark.parametrize( diff --git a/tests/buildtime/gen_spec_v2/test_serialize.py b/tests/buildtime/gen_spec_v2/test_serialize.py index c9fe91c..42234cb 100644 --- a/tests/buildtime/gen_spec_v2/test_serialize.py +++ b/tests/buildtime/gen_spec_v2/test_serialize.py @@ -12,13 +12,11 @@ from toolguard.buildtime.gen_spec_v2.serialize import dump_spec_str, load_spec -from .conftest import requires_corpus +from .conftest import CORPUS_SPECS_DIR -FIXTURE_DIR = Path("tests/data/specs_v2") +FIXTURE_DIR = CORPUS_SPECS_DIR FIXTURES = sorted(FIXTURE_DIR.glob("*.json")) -pytestmark = requires_corpus - def _expected(path: Path) -> str: """The fixture's text with the one deliberate normalization applied. @@ -34,7 +32,9 @@ def _expected(path: Path) -> str: def test_fixtures_are_present(): - assert len(FIXTURES) == 28 + # Guards against a glob that silently matches nothing, which would turn the + # parity gate below into a test that passes by running zero cases. + assert len(FIXTURES) == 6 @pytest.mark.parametrize("path", FIXTURES, ids=lambda p: p.stem) diff --git a/tests/buildtime/gen_spec_v2/test_sysvars.py b/tests/buildtime/gen_spec_v2/test_sysvars.py index a3d9715..2b698e0 100644 --- a/tests/buildtime/gen_spec_v2/test_sysvars.py +++ b/tests/buildtime/gen_spec_v2/test_sysvars.py @@ -8,13 +8,14 @@ import pytest -from .conftest import requires_corpus from toolguard.buildtime.gen_spec_v2.sysvars import ( keep_declared, load_system_vars, render_system_vars, ) +from .conftest import CORPUS_SYSTEM_VARS + EMPLOYEE_VARS = { "user_name": "Bob", "user_id": 1, @@ -102,9 +103,8 @@ def test_a_nested_list_is_still_a_subject_variable(): assert "Platform" in render_system_vars(sv) -@requires_corpus def test_the_real_employee_sys_var_file_yields_only_subject_variables(): - sv = load_system_vars("tests/data/specs_v2_inputs/system_vars.json") + sv = load_system_vars(CORPUS_SYSTEM_VARS) assert sv.names == ["user_name", "user_id", "department", "organization"] diff --git a/tests/examples/employee_mini/README.md b/tests/examples/employee_mini/README.md new file mode 100644 index 0000000..3a36755 --- /dev/null +++ b/tests/examples/employee_mini/README.md @@ -0,0 +1,57 @@ +# employee_mini — a committed slice of the employee ground truth + +The `gen_spec_v2` tests need real generator output to check three things a +synthetic fixture cannot: that the on-disk format round-trips byte for byte, +that references quoted by a model ground back to the policy document they came +from, and that real rule names survive codegen's identifier rules. + +The full employee example — 28 specs, the complete policy document and tool +definitions — lives in the **evaluate-toolguard** project, and benchmarking +belongs there. This directory is the minimum slice that keeps those gates +running in a plain `git clone` of toolguard. + +## Provenance + +Produced from smith's `spec_generation` output for the Enterprise Employee Hub +(`smith/examples/employee/smith/`), with exactly three derivations: + +1. **Six of the 28 specs**, chosen for schema coverage (below), each copied + byte for byte. +2. **`guidance.txt` truncated** to the rules those six specs quote. Seven rule + bullets nothing references were dropped, along with the + `## Administrative Actions` heading both of whose rules went. Every rule + left is one some policy item is grounded in; every kept line is verbatim. +3. **`global.json`'s conflict targets pruned** to the items this subset + contains. It listed eight items belonging to specs that are not here, which + would have left the corpus referring to policy items that do not exist. + +`system_vars.json` is copied unchanged, including the `action_list` and +`action_description` keys — those are the agent's own action catalog, and +`load_system_vars` excluding them is one of the things tested here. + +`tool_definitions.json` is deliberately absent: no test reads it. + +## Why these six specs + +| Spec | Items | What it is here for | +|---|---:|---| +| `update_employee.json` | 7 | both sides of `skip`: three argument-only rules reach codegen, four need system vars or message history | +| `set_passport.json` | 5 | the `missing_variable` → `missing_var` alias that two real fixtures drifted into | +| `create_time_off_request.json` | 5 | a `post_tool` trigger, and a `missing_tool` pending item | +| `global.json` | 3 | all three pending types, and a `tool_name` with no tool behind it | +| `get_employee.json` | 2 | a spec where everything skips, so nothing reaches codegen | +| `list_employees.json` | 0 | the empty spec — "no rule governs this tool" | + +Between them they cover 70 of the 71 distinct key paths in the full 28-spec +corpus. The one absent is a `params` entry inside a `tool_history` record, which +is a free-form dict already exercised by the entries that are here. + +## Numbers the tests assert + +Change any file here and these move with it: + +| Value | Where | +|---|---| +| 6 specs | `test_serialize.py` | +| 27 references, all grounding to themselves | `test_refmatch_real_policy.py` | +| 7 enforceable items across 3 tools | `test_gen_py_contract.py` | diff --git a/tests/examples/employee_mini/guidance.txt b/tests/examples/employee_mini/guidance.txt new file mode 100644 index 0000000..f972b19 --- /dev/null +++ b/tests/examples/employee_mini/guidance.txt @@ -0,0 +1,70 @@ +# Enterprise Employee Hub — Access-Control Policy + +## Context + +The Enterprise Employee Hub exposes an SQLite-backed employee directory +through an MCP agent: employee records, the org chart, departments, personal +records (passport, visa, emergency contact, bank account), country holidays, +and time-off (allotments, requests, and balances). The server performs no +authorization of its own — this policy is the only enforcement layer. The +policy applies based on the acting user's identity and the tool call being +made. + +## Actors and Identity + +The acting user is described by these system variables: + + +- `department` — one of `Corporate Leadership`, `Engineering`, `Product`, `HR`, `Finance`. +- `organization` — one of `IBM Corporation`, `Red Hat`, `Kyndryl`. +- `user_name` — the acting user's display name. +- `user_id` — the acting user's numeric id, matching `employees.user_id`. + +Key derived terms: + +- **Own data** — the target record's `user_id` equals the acting user's `user_id` (self-service). +- **HR** — the acting user's `department` is `HR`. +- **Direct reports** — the employees whose `manager_id` is the acting user's `user_id`. +- **Employee data** — the employee record plus their personal records: home address, passport, visa, emergency contact, and bank account. + +## Data Access + +- An employee may view and edit only their **own data** — home address, passport, visa, emergency contact, and bank account. +- A `Manager` may view only their **direct reports'** data, in addition to their own. +- **HR** may view and edit all employees' data. +- Users outside the IBM organization are strictly prohibited from viewing IBM employee data. +- An employee's `salary` may be updated only by **HR** or by that employee's **direct manager**. + + +## Personal-Record Update Rules + +- An employee may set or update a passport or visa **expiration date** only if the new expiration is more than **six months** after the date of the update. +- An employee may update their **home address** only to an address within their own current country; changing the address to a different country is not allowed through this agent. +- An employee who is on the **blacklist** (persona non grata list) may not update their passport information. +- An employee's passport or visa **issue date** must be strictly earlier than its **expiration date** when both dates are provided in the same call. + +## Data Integrity + +- When an employee's **salary** is set or updated, it must be a positive amount (greater than zero). +- A new or updated employee's work **email** must use their organization's corporate domain: `IBM Corporation` → `@ibm.com`, `Red Hat` → `@redhat.com`, `Kyndryl` → `@kyndryl.com`. This applies when the organization is provided in the same call. + +## Time Off and Leave + +- An employee may not create a time-off request unless they have sufficient available balance for the requested leave type. +- An employee may create a time-off request only for themselves. +- After an employee creates a time-off request, an email is sent to their manager for approval. +- A single time-off request may not span more than **90 consecutive calendar days** (`end_date` minus `start_date`); longer leave must be split into separate requests. + +## Database Writes and Confirmation + +- Before performing any action that writes to or modifies the database(create, update, or delete), the agent must first list the details of the action and obtain the user's explicit confirmation ("yes") before proceeding. +- A user may be deleted only after the requester provides explicit confirmation in exactly this form: + `I request to delete user [USER NAME] with the following user id: [USER ID]`. + +## Agent Behavior + +- The agent should make only one tool call at a time. When it makes a tool call, it should not respond to the user simultaneously; when it responds to the user, it should not make a tool call at the same time. + +## Booking + +- A customer with a `regular` membership may not book a flight for more than three passengers unless they own at least 200 frequent flyer points. diff --git a/tests/examples/employee_mini/specs/create_time_off_request.json b/tests/examples/employee_mini/specs/create_time_off_request.json new file mode 100644 index 0000000..98e4de1 --- /dev/null +++ b/tests/examples/employee_mini/specs/create_time_off_request.json @@ -0,0 +1,157 @@ +{ + "tool_name": "create_time_off_request", + "source_doc": "smith/guidance.txt", + "policy_items": [ + { + "id": "create_time_off_request.own_only", + "name": "An employee may create a time-off request only for themselves", + "description": "A user may create a time-off request only for themselves: the request's user_id (arguments.user_id) must equal the acting user's user_id. If arguments.user_id differs from the acting user's user_id, deny.", + "compliance_examples": [ + "A user creates a time-off request where arguments.user_id equals their own user_id." + ], + "violation_examples": [ + "A user creates a time-off request for another employee (arguments.user_id != subject.user_id).", + "An HR-department user creates a time-off request on behalf of another employee (only the employee themselves may create their request)." + ], + "references": [ + "An employee may create a time-off request only for themselves." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [ + "user_id" + ], + "tool_history": null, + "message_history": null + }, + "resolved_by_user": [ + { + "type": "clarification", + "detail": "The guidance does not state whether an employee may create a time-off request for a user_id other than their own (e.g. HR/manager on behalf of a report).", + "question": "May a user create a time-off request for another employee (arguments.user_id != subject.user_id), or only for themselves?", + "resolution": { + "answer": "Only the employee themselves may create their own time-off request; no on-behalf-of creation (including by HR or a manager).", + "decided_by": "human", + "effect": "Added create_time_off_request.own_only (deny when arguments.user_id != acting user's user_id)." + } + } + ] + }, + { + "id": "create_time_off_request.sufficient_balance", + "name": "Time-off request requires sufficient available balance", + "description": "An employee may not create a time-off request unless they have sufficient available balance for the requested leave type to cover the requested date range. If the number of leave days implied by start_date..end_date exceeds the remaining balance for that leave_type, deny. The remaining balance must be obtained (e.g. via get_leave_balance for the requester and year). Untracked types (e.g. Unpaid) have no balance limit.", + "compliance_examples": [ + "An employee with 10 remaining Vacation days requests 3 Vacation days.", + "An employee requests exactly the number of working days remaining in their balance (still sufficient).", + "An employee requests Unpaid leave, an untracked type with no balance limit." + ], + "violation_examples": [ + "An employee with 2 remaining Vacation days requests 5 Vacation days.", + "An employee with 0 remaining days for the leave type requests any days of that type.", + "The requested date range (after excluding weekends and the employee's country holidays) exceeds the remaining balance." + ], + "references": [ + "An employee may not create a time-off request unless they have sufficient available balance for the requested leave type." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [], + "tool_history": [ + { + "tool": "get_leave_balance", + "params": { + "user_id": "input.arguments.user_id", + "year": "year-of(input.arguments.start_date)" + } + } + ], + "message_history": null + } + }, + { + "id": "create_time_off_request.notify_manager_after_create", + "name": "Email the manager for approval after a request is created", + "description": "After an employee creates a time-off request, an email must be sent to their manager for approval. This is a required post-action side effect rather than an allow/deny condition on the create itself.", + "compliance_examples": [ + "After the request is created, an approval email is dispatched to the employee's manager." + ], + "violation_examples": [ + "A time-off request is created but no approval notification is sent to the manager.", + "The request is created for an employee who has no manager (top of the org) and it is unclear who receives the approval email." + ], + "references": [ + "After an employee creates a time-off request, an email is sent to their manager for approval." + ], + "trigger": "post_tool", + "requires": { + "system_vars": [], + "tool_history": null, + "message_history": null + }, + "pending_for_user": [ + { + "type": "missing_tool", + "detail": "There is no email / notification tool exposed by the MCP server, so the required 'email the manager for approval' side effect cannot be performed or verified.", + "suggested_tool": "send_email / notify_manager", + "question": "Which tool should send the manager approval email after a time-off request is created? Should a notification tool be added, or is this handled outside the agent?" + } + ] + }, + { + "id": "create_time_off_request.max_length_90_days", + "name": "A single time-off request may not exceed 90 calendar days", + "description": "The requested date range must not exceed 90 consecutive calendar days. If the span from arguments.start_date to arguments.end_date is more than 90 calendar days, deny. This is a plain calendar-day span computed from start_date and end_date; it is NOT a business-day count, so no holiday lookup is required.", + "compliance_examples": [ + "start_date 2026-08-01 and end_date 2026-08-15 (15 calendar days, well within 90).", + "start_date 2026-08-01 and end_date 2026-09-30 (about 60 calendar days, within 90)." + ], + "violation_examples": [ + "start_date 2026-08-01 and end_date 2026-12-01 (about 122 calendar days, exceeds 90).", + "start_date 2026-01-01 and end_date 2026-12-31 (a full year, far exceeding 90 days)." + ], + "references": [ + "A single time-off request may not span more than **90 consecutive calendar days** (`end_date` minus `start_date`); longer leave must be split into separate requests." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [], + "tool_history": null, + "message_history": null + } + }, + { + "id": "create_time_off_request.confirm_before_write", + "name": "Require explicit confirmation before creating a time-off request", + "description": "create_time_off_request writes to the database. Before it runs, the agent must first list the details of the request (leave_type, start_date, end_date) to the user and obtain the user's explicit 'yes' confirmation. If the conversation history contains no explicit user confirmation ('yes') for this action, deny.", + "compliance_examples": [ + "The agent lists the exact details of the action to create the time-off request, the user replies 'yes', and only then is the tool called.", + "The user explicitly approves this specific action ('yes, go ahead') before the tool to create the time-off request is called." + ], + "violation_examples": [ + "The tool to create the time-off request is called with no prior confirmation step at all.", + "The agent presents the action but calls the tool to create the time-off request before the user responds.", + "The user replies with something ambiguous ('sounds good', 'ok maybe') rather than an explicit 'yes'.", + "The user previously said 'yes' to a different action; there is no explicit 'yes' for this specific action to create the time-off request.", + "The user explicitly declines ('no'), but the tool to create the time-off request is called anyway." + ], + "references": [ + "Before performing any action that writes to or modifies the database(create, update, or delete), the agent must first list the details of the action and obtain the user's explicit confirmation (\"yes\") before proceeding." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [], + "tool_history": null, + "message_history": true + } + } + ], + "debug": { + "tool_info": { + "is_read_only": false, + "user_enrichment": "Creates a Pending time-off request. Balance depends on get_leave_balance; a manager-approval email is expected afterward." + }, + "archive": [], + "notes": [] + } +} diff --git a/tests/examples/employee_mini/specs/get_employee.json b/tests/examples/employee_mini/specs/get_employee.json new file mode 100644 index 0000000..1992e86 --- /dev/null +++ b/tests/examples/employee_mini/specs/get_employee.json @@ -0,0 +1,79 @@ +{ + "tool_name": "get_employee", + "source_doc": "smith/guidance.txt", + "policy_items": [ + { + "id": "get_employee.read_own_report_or_hr", + "name": "Employee-record read limited to self, direct reports, or HR", + "description": "A user may read an employee record only if: it is their own (arguments.user_id equals the acting user's user_id); the acting user is in the HR department (HR may view all); or the target is one of the acting user's direct reports (the target's manager_id equals the acting user's user_id). Otherwise deny. Whether the acting user manages the target is determined by the reporting relationship (get_direct_reports for the acting user, or the target's manager_id from get_employee), not by any role variable.", + "compliance_examples": [ + "A user reads their own employee record (arguments.user_id == subject.user_id).", + "An HR-department user reads any employee's employee record.", + "A user reads a direct report's employee record (the report's manager_id equals the acting user's user_id)." + ], + "violation_examples": [ + "A non-HR user reads the employee record of a colleague who is neither themselves nor one of their direct reports.", + "A user who is not in HR reads the employee record of an employee who does not report to them." + ], + "references": [ + "An employee may view and edit only their **own data** — home address, passport, visa, emergency contact, and bank account.", + "A `Manager` may view only their **direct reports'** data, in addition to their own.", + "**HR** may view and edit all employees' data." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [ + "user_id", + "department" + ], + "tool_history": [ + { + "tool": "get_direct_reports", + "params": { + "user_id": "input.extensions.subject.user_id" + } + } + ], + "message_history": null + } + }, + { + "id": "get_employee.outside_ibm_no_ibm_data", + "name": "Users outside the IBM organization may not view IBM employee data", + "description": "If the acting user's organization is not 'IBM Corporation' (the user is outside the IBM organization), they may not view data of an employee who belongs to 'IBM Corporation'. Deny the read when the acting user's organization is not 'IBM Corporation' and the target employee is an IBM (IBM Corporation) employee.", + "compliance_examples": [ + "An IBM Corporation user accesses an IBM employee's employee record (the acting user is inside the IBM organization).", + "A Red Hat user accesses a Red Hat employee's employee record (the target is not an IBM employee)." + ], + "violation_examples": [ + "A Red Hat user accesses an IBM employee's employee record (a user outside the IBM organization accessing IBM employee data).", + "A Kyndryl user accesses an IBM employee's employee record (another organization outside IBM accessing IBM employee data)." + ], + "references": [ + "Users outside the IBM organization are strictly prohibited from viewing IBM employee data." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [ + "organization" + ], + "tool_history": [ + { + "tool": "get_employee", + "params": { + "user_id": "input.arguments.user_id" + } + } + ], + "message_history": null + } + } + ], + "debug": { + "tool_info": { + "is_read_only": true, + "user_enrichment": "Returns the full employee record, including salary and home_address." + }, + "archive": [] + } +} diff --git a/tests/examples/employee_mini/specs/global.json b/tests/examples/employee_mini/specs/global.json new file mode 100644 index 0000000..7bd1575 --- /dev/null +++ b/tests/examples/employee_mini/specs/global.json @@ -0,0 +1,114 @@ +{ + "tool_name": "global", + "source_doc": "smith/guidance.txt", + "policy_items": [ + { + "id": "global.delete_user_requires_exact_confirmation", + "name": "Deleting a user requires an exact confirmation phrase", + "description": "A user (employee) may be deleted only after the requester provides explicit confirmation in exactly this form: 'I request to delete user [USER NAME] with the following user id: [USER ID]', with the name and id substituted for the user being deleted. This rule cannot be attached to a single tool because the MCP server exposes no employee-deletion tool (only delete_holiday deletes holidays, not users).", + "compliance_examples": [ + "The requester types exactly 'I request to delete user Jane Doe with the following user id: 42', with a name and id that match the target user, and only then is the deletion carried out." + ], + "violation_examples": [ + "The requester says 'yes, delete Jane' or 'please remove user 42' without the exact required phrase.", + "The exact phrase is used but the user name does not match the given user id.", + "The exact phrase is used but the user id does not match the named user.", + "No confirmation phrase is provided at all.", + "Any employee deletion is attempted, since no employee-deletion tool exists to enforce this against." + ], + "references": [ + "A user may be deleted only after the requester provides explicit confirmation in exactly this form:", + "`I request to delete user [USER NAME] with the following user id: [USER ID]`." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [], + "tool_history": null, + "message_history": true + }, + "pending_for_user": [ + { + "type": "missing_tool", + "detail": "The guidance defines a delete-user confirmation rule, but there is no tool in tool_definitions.json that deletes an employee/user. The rule therefore has no tool to attach to.", + "suggested_tool": "delete_employee", + "question": "Which tool is intended to delete a user? Should a delete_employee tool be added so this confirmation rule can be enforced, or is user deletion out of scope for this agent?" + } + ] + }, + { + "id": "global.single_tool_call_at_a_time", + "name": "Agent must not interleave tool calls and user responses", + "description": "The agent must make only one tool call at a time. When it makes a tool call it must not also respond to the user in the same turn; when it responds to the user it must not also make a tool call in the same turn. This is an orchestration constraint on agent turn structure, not a condition on any single tool's arguments, so it cannot be attached to one tool.", + "compliance_examples": [ + "In one turn the agent emits a single tool call and no user-facing text.", + "In one turn the agent replies to the user and makes no tool call." + ], + "violation_examples": [ + "The agent emits two or more tool calls in the same turn.", + "The agent emits a tool call and a user-facing message in the same turn." + ], + "references": [ + "The agent should make only one tool call at a time. When it makes a tool call, it should not respond to the user simultaneously; when it responds to the user, it should not make a tool call at the same time." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [], + "tool_history": null, + "message_history": true + }, + "pending_for_user": [ + { + "type": "clarification", + "detail": "This is a turn-structure / orchestration rule about the agent loop rather than an allow/deny condition on any one tool's arguments. It is unclear whether it should be enforced as an OPA policy at all versus in the agent runtime.", + "question": "Should the 'one tool call at a time, never interleaved with a user response' rule be enforced by the policy engine, or handled in the agent orchestration layer?" + } + ] + }, + { + "id": "global.flight_booking_passenger_limit", + "name": "Flight booking passenger limit (foreign-domain rule)", + "description": "The guidance contains a rule that a customer with a 'regular' membership may not book a flight for more than three passengers unless they own at least 200 frequent flyer points. This rule belongs to a flight-booking domain and does not match the Enterprise Employee Hub: there is no flight-booking tool, no 'membership' concept, and no 'frequent flyer points' variable anywhere in the tools or system variables. It appears to be a leftover from a different policy document.", + "compliance_examples": [], + "violation_examples": [], + "references": [ + "A customer with a `regular` membership may not book a flight for more than three passengers unless they own at least 200 frequent flyer points." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [], + "tool_history": null, + "message_history": null + }, + "pending_for_user": [ + { + "type": "clarification", + "detail": "This rule references flight booking, memberships, passengers, and frequent flyer points — none of which exist in this employee-hub MCP server, its tools, or its system variables. It conflicts with the actual domain of this policy.", + "question": "This flight-booking rule does not belong to the employee hub. Should it be removed from guidance.txt, or is a flight-booking tool/domain expected to be added?" + } + ] + } + ], + "conflicts": [ + { + "id": "conflict.global.ibm_organization_definition", + "name": "What counts as 'the IBM organization' for the outside-IBM prohibition?", + "kind": "definition", + "conflicting_policies": [ + "get_employee.outside_ibm_no_ibm_data", + "get_employee.read_own_report_or_hr", + "update_employee.outside_ibm_no_ibm_data", + "update_employee.edit_own_or_hr" + ], + "description": "Applies across every personal/employee read and edit tool whenever an HR user or a manager whose organization is Red Hat or Kyndryl accesses an IBM employee's record. The role grant (read_own_report_or_hr / edit_own_or_hr) would allow the access, while outside_ibm_no_ibm_data denies any access to IBM-Corporation employee data by a user whose organization is not 'IBM Corporation'. Under AND composition the prohibition wins, but the compiled policy hardcodes organization == 'IBM Corporation' and so treats Red Hat and Kyndryl — both IBM-owned — as 'outside IBM'. Whether that is intended is a data-model/definition question that spans all these tools, not a per-tool dominance decision, so it is recorded once here.", + "question": "Does 'the IBM organization' mean only organization == 'IBM Corporation' (so Red Hat and Kyndryl users are 'outside IBM' and blocked from IBM employee data), or the whole IBM group including Red Hat and Kyndryl? If the latter, the outside_ibm policies need a group definition rather than a single-organization equality check.", + "resolution": null + } + ], + "debug": { + "tool_info": { + "is_read_only": false, + "user_enrichment": "Global bucket for rules that cannot be attached to a single tool: orphan rules (no matching tool), orchestration-level rules, foreign-domain conflicts, and cross-cutting missing-data gaps." + }, + "archive": [] + } +} diff --git a/tests/examples/employee_mini/specs/list_employees.json b/tests/examples/employee_mini/specs/list_employees.json new file mode 100644 index 0000000..17a42de --- /dev/null +++ b/tests/examples/employee_mini/specs/list_employees.json @@ -0,0 +1,64 @@ +{ + "tool_name": "list_employees", + "source_doc": "smith/guidance.txt", + "policy_items": [], + "debug": { + "tool_info": { + "is_read_only": true, + "user_enrichment": "Bulk read returning org-directory fields only (user_id, first_name, last_name, role, organization, title, department_id, manager_id, country_code). Salary, home_address, email, salary_currency, and start_date are NOT returned. Filter args: department_id, manager_id, country_code." + }, + "archive": [ + { + "id": "list_employees.restrict_bulk_read", + "name": "Bulk employee listing limited to authorized scope", + "description": "list_employees is a bulk read that returns employee records (including salary) for many employees. A non-HR user may only receive employees they are authorized to view: their own record, and — for a Manager — their direct reports. HR may list all employees. Deny a listing whose scope would expose employees the acting user is not authorized to view.", + "compliance_examples": [ + "An HR-department user lists all employees.", + "A Manager lists employees filtered to manager_id equal to their own user_id (their direct reports).", + "An IC lists employees filtered to their own user_id only." + ], + "violation_examples": [ + "An IC lists all employees with no restricting filter.", + "A Manager lists employees filtered to a department or manager_id they do not manage.", + "A non-HR user issues an unfiltered list that would return employees outside their authorized scope.", + "A Manager lists employees filtered by manager_id equal to a different manager's id." + ], + "references": [ + "An employee may view and edit only their **own data** — home address, passport, visa, emergency contact, and bank account.", + "A `Manager` may view only their **direct reports'** data, in addition to their own.", + "**HR** may view and edit all employees' data." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [ + "user_id", + "department", + "role" + ], + "tool_history": [ + { + "tool": "get_direct_reports", + "params": { + "user_id": "input.extensions.subject.user_id" + } + } + ], + "message_history": null + }, + "resolved_by_user": [ + { + "type": "clarification", + "detail": "The data-access rules are phrased per-employee, but list_employees returns many rows and its authorized scope depends on the filter arguments. It is ambiguous whether the guard should deny the call up-front (pre_tool) based on filters, or filter the returned rows (post_tool).", + "question": "For bulk listing, should the policy deny non-HR callers unless they scope the query to themselves/their direct reports (pre_tool), or should it filter out unauthorized rows from the result (post_tool)?", + "resolution": { + "answer": "Moot: list_employees was changed to return only non-sensitive org-directory fields (no salary, home_address, or other PII), so bulk listing no longer exposes protected employee data. The scoping restriction is dropped entirely — any authenticated user may list employees — so neither pre_tool scoping nor post_tool row filtering is needed.", + "decided_by": "human", + "effect": "Rule removed from active policy_items and archived; list_employees is unrestricted." + } + } + ], + "archive_reason": "list_employees was changed (api/employees.py) to return only non-sensitive org-directory fields (user_id, first_name, last_name, role, organization, title, department_id, manager_id, country_code); salary and all PII (home_address, email, salary_currency, start_date) were removed. With no sensitive data exposed by the tool, the HR-all / Manager-reports / IC-self scoping is no longer warranted, so the rule is dropped and the tool is available to any authenticated user." + } + ] + } +} diff --git a/tests/examples/employee_mini/specs/set_passport.json b/tests/examples/employee_mini/specs/set_passport.json new file mode 100644 index 0000000..35e7a0e --- /dev/null +++ b/tests/examples/employee_mini/specs/set_passport.json @@ -0,0 +1,153 @@ +{ + "tool_name": "set_passport", + "source_doc": "smith/guidance.txt", + "policy_items": [ + { + "id": "set_passport.edit_own_or_hr", + "name": "Setting a passport limited to self or HR", + "description": "A user may create/replace an employee's passport only if it is their own (arguments.user_id equals the acting user's user_id) or the acting user is in the HR department (HR may edit all employees' data). Managers may only view, not edit, their reports' personal data. Deny writing another employee's passport by a non-HR user.", + "compliance_examples": [ + "A user creates/updates their own passport (arguments.user_id == subject.user_id).", + "An HR-department user creates/updates another employee's passport." + ], + "violation_examples": [ + "A non-HR user (department != HR) creates/updates another employee's passport (arguments.user_id != subject.user_id).", + "A user creates/updates the passport of one of their direct reports (the target's manager_id equals the acting user's user_id) — a direct manager may view a report's data but not edit it." + ], + "references": [ + "An employee may view and edit only their **own data** — home address, passport, visa, emergency contact, and bank account.", + "**HR** may view and edit all employees' data." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [ + "user_id", + "department" + ], + "tool_history": null, + "message_history": null + } + }, + { + "id": "set_passport.expiry_min_six_months", + "name": "Passport expiration must be more than six months out", + "description": "If expiry_date is provided, the new expiration must be more than six months after the current date (the date of the update). If the provided expiry_date is not more than six months in the future, deny. The current date is supplied by the runtime clock.", + "compliance_examples": [ + "On 2026-07-16 the passport expiry_date is set to 2028-01-01 (well more than six months out).", + "On 2026-07-16 the passport expiry_date is set to 2027-02-01 (more than six months out).", + "The call updates other passport fields without providing an expiry_date, so this rule does not apply." + ], + "violation_examples": [ + "On 2026-07-16 the passport expiry_date is set to 2027-01-16 (exactly six months out — the rule requires MORE than six months).", + "On 2026-07-16 the passport expiry_date is set to 2026-09-01 (less than six months out).", + "The passport expiry_date is set to a date in the past." + ], + "references": [ + "An employee may set or update a passport or visa **expiration date** only if the new expiration is more than **six months** after the date of the update." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [], + "tool_history": null, + "message_history": null + } + }, + { + "id": "set_passport.blacklist_blocks_update", + "name": "Blacklisted employees may not set passport information", + "description": "An employee who is on the blacklist (persona non grata list) may not create or update their passport information. If the acting user is on the blacklist, deny.", + "compliance_examples": [ + "A non-blacklisted employee updates their own passport information." + ], + "violation_examples": [ + "A blacklisted (persona non grata) employee updates their passport information.", + "A blacklisted employee sets a brand-new passport record." + ], + "references": [ + "An employee who is on the **blacklist** (persona non grata list) may not update their passport information." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [], + "tool_history": null, + "message_history": null + }, + "pending_for_user": [ + { + "type": "missing_variable", + "detail": "No blacklist / persona-non-grata signal exists in system_vars.json and no tool returns blacklist status. See global.blacklist_membership_source.", + "suggested_source": "system_vars.json:blacklist or a get_blacklist_status tool", + "question": "How does the policy learn whether an employee is on the blacklist?" + } + ] + }, + { + "id": "set_passport.issue_before_expiry", + "name": "Passport issue date must precede expiration date", + "description": "When both issue_date and expiry_date are provided in the same call, issue_date must be strictly earlier than expiry_date. If issue_date is on or after expiry_date, deny. When only one of the two dates is provided, this rule does not apply (the stored counterpart is not looked up).", + "compliance_examples": [ + "issue_date 2026-01-01 and expiry_date 2036-01-01 (issue strictly before expiry).", + "The call provides only an expiry_date (or only an issue_date), so this rule does not apply." + ], + "violation_examples": [ + "issue_date 2030-01-01 and expiry_date 2026-01-01 (issue after expiry).", + "issue_date 2026-01-01 and expiry_date 2026-01-01 (equal dates — issue must be strictly earlier)." + ], + "references": [ + "An employee's passport or visa **issue date** must be strictly earlier than its **expiration date** when both dates are provided in the same call." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [], + "tool_history": null, + "message_history": null + } + }, + { + "id": "set_passport.confirm_before_write", + "name": "Require explicit confirmation before setting a passport", + "description": "set_passport writes to the database. Before it runs, the agent must first list the details of the action to the user and obtain the user's explicit 'yes' confirmation. If the conversation history contains no explicit user confirmation ('yes') for this action, deny.", + "compliance_examples": [ + "The agent lists the exact details of the action to set the passport, the user replies 'yes', and only then is the tool called.", + "The user explicitly approves this specific action ('yes, go ahead') before the tool to set the passport is called." + ], + "violation_examples": [ + "The tool to set the passport is called with no prior confirmation step at all.", + "The agent presents the action but calls the tool to set the passport before the user responds.", + "The user replies with something ambiguous ('sounds good', 'ok maybe') rather than an explicit 'yes'.", + "The user previously said 'yes' to a different action; there is no explicit 'yes' for this specific action to set the passport.", + "The user explicitly declines ('no'), but the tool to set the passport is called anyway." + ], + "references": [ + "Before performing any action that writes to or modifies the database(create, update, or delete), the agent must first list the details of the action and obtain the user's explicit confirmation (\"yes\") before proceeding." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [], + "tool_history": null, + "message_history": true + } + } + ], + "conflicts": [ + { + "id": "conflict.set_passport.blacklist_scope_vs_hr", + "name": "Does the blacklist passport block bind HR setting others'?", + "kind": "scope", + "conflicting_policies": [ + "set_passport.edit_own_or_hr", + "set_passport.blacklist_blocks_update" + ], + "description": "Applies when HR sets/upserts the passport of a blacklisted employee. edit_own_or_hr grants HR edit access to all employees' passports, while blacklist_blocks_update denies passport writes connected to the blacklist. The guidance says 'An employee who is on the blacklist may not update their passport information', which reads as a constraint on the blacklisted person acting on their own record. It is unclear whose blacklist status matters — the acting user's or the target record owner's — and therefore whether HR (not blacklisted) may set a blacklisted employee's passport on their behalf.", + "question": "Does the blacklist block key off the acting user (only a blacklisted user is blocked, so HR is unaffected) or the target record owner (any passport write for a blacklisted employee is blocked, including by HR)?", + "resolution": null + } + ], + "debug": { + "tool_info": { + "is_read_only": false, + "user_enrichment": "Upserts sensitive PII (passport). Subject to own-data edit rule, expiry rule, and blacklist rule." + }, + "archive": [] + } +} diff --git a/tests/examples/employee_mini/specs/update_employee.json b/tests/examples/employee_mini/specs/update_employee.json new file mode 100644 index 0000000..de7e2ab --- /dev/null +++ b/tests/examples/employee_mini/specs/update_employee.json @@ -0,0 +1,230 @@ +{ + "tool_name": "update_employee", + "source_doc": "smith/guidance.txt", + "policy_items": [ + { + "id": "update_employee.edit_own_or_hr", + "name": "Editing an employee record is limited to self or HR", + "description": "A user may edit an employee record only if it is their own (arguments.user_id equals the acting user's user_id) or the acting user is in the HR department (HR may edit all employees). A Manager may only view — not edit — their direct reports, so managers get no edit access to another employee's record here. Deny any edit of another employee's record by a non-HR user.", + "compliance_examples": [ + "A user creates/updates their own employee record (arguments.user_id == subject.user_id).", + "An HR-department user creates/updates another employee's employee record." + ], + "violation_examples": [ + "A non-HR user (department != HR) creates/updates another employee's employee record (arguments.user_id != subject.user_id).", + "A user creates/updates the employee record of one of their direct reports (the target's manager_id equals the acting user's user_id) — a direct manager may view a report's data but not edit it." + ], + "references": [ + "An employee may view and edit only their **own data** — home address, passport, visa, emergency contact, and bank account.", + "**HR** may view and edit all employees' data." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [ + "user_id", + "department" + ], + "tool_history": null, + "message_history": null + } + }, + { + "id": "update_employee.salary_by_hr_or_direct_manager", + "name": "Salary changes restricted to HR or the direct manager", + "description": "If the call sets or changes the salary field, the acting user must be in the HR department or be the target employee's direct manager (the target employee's manager_id equals the acting user's user_id). Otherwise deny. The target's manager_id must be looked up (e.g. via get_employee) to evaluate the direct-manager condition.", + "compliance_examples": [ + "An HR-department user updates an employee's salary.", + "A user updates the salary of a direct report (the employee's manager_id equals the acting user's user_id).", + "A user updates non-salary fields only (the rule does not fire when salary is unchanged)." + ], + "violation_examples": [ + "A non-HR user who is not the target's direct manager updates the target's salary (the target's manager_id != the acting user's user_id) — even a manager may change salary only for their own direct reports.", + "A non-HR user updates their own salary (self-service does not authorize a salary change; only HR or the employee's direct manager may)." + ], + "references": [ + "An employee's `salary` may be updated only by **HR** or by that employee's **direct manager**." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [ + "department", + "user_id" + ], + "tool_history": [ + { + "tool": "get_employee", + "params": { + "user_id": "input.arguments.user_id" + } + } + ], + "message_history": null + } + }, + { + "id": "update_employee.home_address_same_country", + "name": "Home-address changes must stay within the current country", + "description": "When home_address is updated, the change must not move the employee to a different country. The country of the employee is tracked by the country_code field: the employee's current country_code (looked up via get_employee) is compared against the country_code after the update. If the update changes country_code to a different country, the move is cross-country and is denied. A home_address change that leaves country_code unchanged stays within the same country and is allowed.", + "compliance_examples": [ + "An employee whose current country_code is 'US' updates their home_address and keeps country_code 'US'.", + "The update does not change country_code (the employee stays in the same country).", + "The update changes fields other than home_address / country_code (the rule does not apply)." + ], + "violation_examples": [ + "An employee whose current country_code is 'US' changes country_code to 'CA' (a cross-country move).", + "A home_address update accompanied by a country_code change to a different country than the employee's current country_code." + ], + "references": [ + "An employee may update their **home address** only to an address within their own current country; changing the address to a different country is not allowed through this agent." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [], + "tool_history": [ + { + "tool": "get_employee", + "params": { + "user_id": "input.arguments.user_id" + } + } + ], + "message_history": null + }, + "resolved_by_user": [ + { + "type": "clarification", + "detail": "home_address is a free-text string with no separate country field, and country_code is a distinct field. Determining the country of a new free-text address (to compare against the current country) is ambiguous.", + "suggested_source": "get_employee (current country_code)", + "question": "How should the policy determine the country of a new home_address? Should address changes also update country_code, or should the agent supply a structured country for the new address?", + "resolution": { + "answer": "We can determine whether the employee moved to another country by the country_code field: compare the updated country_code against the employee's current country_code (from get_employee). If the country_code changed to a different country, it is a cross-country move and is denied; the free-text home_address itself is not parsed for a country.", + "decided_by": "human", + "effect": "Zone 1 updated: cross-country moves are detected by comparing the updated country_code against the current country_code (via get_employee) rather than inferring the country from the free-text home_address." + } + } + ] + }, + { + "id": "update_employee.outside_ibm_no_ibm_data", + "name": "Users outside the IBM organization may not access IBM employee data", + "description": "If the acting user's organization is not 'IBM Corporation' (the user is outside the IBM organization), they may not view or edit data of an employee who belongs to 'IBM Corporation'. Deny the update when the acting user's organization is not 'IBM Corporation' and the target employee is an IBM (IBM Corporation) employee.", + "compliance_examples": [ + "An IBM Corporation user accesses an IBM employee's employee record (the acting user is inside the IBM organization).", + "A Red Hat user accesses a Red Hat employee's employee record (the target is not an IBM employee)." + ], + "violation_examples": [ + "A Red Hat user accesses an IBM employee's employee record (a user outside the IBM organization accessing IBM employee data).", + "A Kyndryl user accesses an IBM employee's employee record (another organization outside IBM accessing IBM employee data)." + ], + "references": [ + "Users outside the IBM organization are strictly prohibited from viewing IBM employee data." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [ + "organization" + ], + "tool_history": [ + { + "tool": "get_employee", + "params": { + "user_id": "input.arguments.user_id" + } + } + ], + "message_history": null + } + }, + { + "id": "update_employee.salary_positive", + "name": "Salary must be positive when set or updated", + "description": "When a salary value is provided in the update, it must be a positive amount (greater than zero). If salary is provided and is zero or negative, deny. When the update does not touch salary, this rule does not apply. This is a value-integrity check distinct from the authorization rule restricting who may change a salary.", + "compliance_examples": [ + "An employee's salary is updated to 120000.", + "The update changes non-salary fields only (the rule does not apply)." + ], + "violation_examples": [ + "An employee's salary is updated to 0.", + "An employee's salary is updated to -1000." + ], + "references": [ + "When an employee's **salary** is set or updated, it must be a positive amount (greater than zero)." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [], + "tool_history": null, + "message_history": null + } + }, + { + "id": "update_employee.email_matches_org_domain", + "name": "Employee email must match the organization's corporate domain", + "description": "When both email and organization are provided in the same update call, the email domain must be the corporate domain mapped to that organization: 'IBM Corporation' -> ibm.com, 'Red Hat' -> redhat.com, 'Kyndryl' -> kyndryl.com. If the email domain does not match the mapped domain for the provided organization, deny. When the call does not include organization (e.g. an email-only update), this rule does not apply, because the record's organization would have to be looked up (tool history), which is intentionally out of scope.", + "compliance_examples": [ + "The update sets organization 'IBM Corporation' and email 'alice@ibm.com'.", + "The update sets organization 'Red Hat' and email 'bob@redhat.com'.", + "The update changes email but does not include organization, so this rule does not apply." + ], + "violation_examples": [ + "The update sets organization 'IBM Corporation' and email 'alice@gmail.com'.", + "The update sets organization 'Kyndryl' and email 'carol@ibm.com'." + ], + "references": [ + "A new or updated employee's work **email** must use their organization's corporate domain: `IBM Corporation` → `@ibm.com`, `Red Hat` → `@redhat.com`, `Kyndryl` → `@kyndryl.com`. This applies when the organization is provided in the same call." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [], + "tool_history": null, + "message_history": null + } + }, + { + "id": "update_employee.confirm_before_write", + "name": "Require explicit confirmation before updating the employee", + "description": "update_employee modifies the database. Before it runs, the agent must first list the details of the change to the user and obtain the user's explicit 'yes' confirmation. If the conversation history contains no explicit user confirmation ('yes') for this specific update, deny.", + "compliance_examples": [ + "The agent lists the exact details of the action to update the employee record, the user replies 'yes', and only then is the tool called.", + "The user explicitly approves this specific action ('yes, go ahead') before the tool to update the employee record is called." + ], + "violation_examples": [ + "The tool to update the employee record is called with no prior confirmation step at all.", + "The agent presents the action but calls the tool to update the employee record before the user responds.", + "The user replies with something ambiguous ('sounds good', 'ok maybe') rather than an explicit 'yes'.", + "The user previously said 'yes' to a different action; there is no explicit 'yes' for this specific action to update the employee record.", + "The user explicitly declines ('no'), but the tool to update the employee record is called anyway." + ], + "references": [ + "Before performing any action that writes to or modifies the database(create, update, or delete), the agent must first list the details of the action and obtain the user's explicit confirmation (\"yes\") before proceeding." + ], + "trigger": "pre_tool", + "requires": { + "system_vars": [], + "tool_history": null, + "message_history": true + } + } + ], + "conflicts": [ + { + "id": "conflict.update_employee.home_address_scope_vs_hr", + "name": "Does the in-country home-address rule bind HR editing others?", + "kind": "scope", + "conflicting_policies": [ + "update_employee.edit_own_or_hr", + "update_employee.home_address_same_country" + ], + "description": "Applies when HR updates another employee's home_address to an address in a different country (e.g. an approved relocation). edit_own_or_hr grants HR edit access to all employees, while home_address_same_country denies any change that moves the employee's country_code. Because a tool's policy items AND together, the deny wins and HR is blocked from cross-country relocations. But the guidance phrases the in-country rule as an employee self-service constraint ('An employee may update their home address only ...'), so it is unclear whether it is meant to bind HR acting on another employee's record at all.", + "question": "Does 'home address must stay within the current country' apply to HR (and managers) editing another employee's record, or only to an employee editing their own? If HR-initiated relocations are legitimate, home_address_same_country should be scoped to self-service edits.", + "resolution": null + } + ], + "debug": { + "tool_info": { + "is_read_only": false, + "user_enrichment": "Can change privilege-sensitive fields (role, manager_id, department_id, salary) and home_address." + }, + "archive": [] + } +} diff --git a/tests/examples/employee_mini/system_vars.json b/tests/examples/employee_mini/system_vars.json new file mode 100644 index 0000000..000fbf1 --- /dev/null +++ b/tests/examples/employee_mini/system_vars.json @@ -0,0 +1,78 @@ +{ + "action_list": [ + "add_employee", + "update_employee", + "get_employee", + "list_employees", + "get_manager", + "get_direct_reports", + "get_reporting_chain", + "add_department", + "update_department", + "get_department", + "list_departments", + "set_passport", + "update_passport", + "get_passport", + "set_visa", + "update_visa", + "get_visa", + "set_emergency_contact", + "update_emergency_contact", + "get_emergency_contact", + "set_bank_account", + "update_bank_account", + "get_bank_account", + "set_leave_allotment", + "get_leave_allotments", + "create_time_off_request", + "update_time_off_status", + "get_time_off_request", + "list_time_off_requests", + "get_leave_balance", + "add_holiday", + "list_holidays", + "delete_holiday", + "other" + ], + "action_description": { + "add_employee": "Create a new employee record (name, email, role, title, home_address, country_code; optional department_id, manager_id, salary, salary_currency, start_date).", + "update_employee": "Update any provided fields on an existing employee, including role, title, department, manager, salary, and home_address.", + "get_employee": "Return the full employee record for a given user_id, including salary and home_address.", + "list_employees": "List employees, optionally filtered by department_id, manager_id, or country_code.", + "get_manager": "Return the manager of a given employee.", + "get_direct_reports": "Return the employees who report directly to a given user_id.", + "get_reporting_chain": "Return the chain of managers from an employee up to the top of the org.", + "add_department": "Create a new department with a unique name and optional description.", + "update_department": "Update a department's name and/or description.", + "get_department": "Return a single department record.", + "list_departments": "Return all departments.", + "set_passport": "Create or replace an employee's passport record (passport number, issuing country, issue/expiry dates).", + "update_passport": "Update provided fields on an employee's passport record.", + "get_passport": "Return an employee's passport record (sensitive PII).", + "set_visa": "Create or replace an employee's visa record (visa number, type, issuing country, dates).", + "update_visa": "Update provided fields on an employee's visa record.", + "get_visa": "Return an employee's visa record (sensitive PII).", + "set_emergency_contact": "Create or replace an employee's emergency contact (name, relationship, phone, address).", + "update_emergency_contact": "Update provided fields on an employee's emergency contact.", + "get_emergency_contact": "Return an employee's emergency contact record (sensitive PII).", + "set_bank_account": "Create or replace an employee's bank account (bank name, account number, routing number, IBAN, currency).", + "update_bank_account": "Update provided fields on an employee's bank account.", + "get_bank_account": "Return an employee's bank account record (highly sensitive PII).", + "set_leave_allotment": "Set the annual day allotment for a leave type (Vacation, Sick Leave, Maternity, Paternity, Jury Duty, Unpaid).", + "get_leave_allotments": "Return all leave allotments for an employee.", + "create_time_off_request": "Create a Pending time-off request for an employee over a date range and leave type.", + "update_time_off_status": "Set a time-off request's status to Pending, Approved, or Denied (approval action).", + "get_time_off_request": "Return a single time-off request by request_id.", + "list_time_off_requests": "List time-off requests, optionally filtered by user_id and/or status.", + "get_leave_balance": "Return per-leave-type balance (allotment, used, remaining) for an employee and year.", + "add_holiday": "Add a country holiday (country_code, date, name).", + "list_holidays": "List holidays for a country, optionally filtered to a year.", + "delete_holiday": "Delete a holiday by holiday_id.", + "other": "Other general Q&A not backed by a specific tool." + }, + "user_name": "Bob", + "user_id": 1, + "department": ["Corporate Leadership", "Engineering", "Product", "HR", "Finance"], + "organization": ["IBM Corporation", "Red Hat", "Kyndryl"] +} From fd3e570aa6562283ac407d18d0b3ac264577e36a Mon Sep 17 00:00:00 2001 From: naamaz Date: Wed, 12 Aug 2026 08:48:40 +0300 Subject: [PATCH 4/6] Recover JSON replies with bare inner quotes instead of aborting the run A policy document that quotes a literal -- obtain the user's explicit confirmation ("yes") -- makes models escape the inner quotes in one field and leave them bare in another, in the same reply. The payload is complete and brace-balanced; only the parse fails. The retry loop could never fix that. It re-sent an unchanged message list, so the model had no new information and reproduced the same mistake on every attempt: five attempts were one attempt billed five times. Make the retry a repair turn instead -- the rejected reply and the parser's own error text go back to the model, which knows which quote was content. That covers every malformation class, not just quotes. Two consequences of fixing the loop: - The backoff sleep between attempts is gone. It bought nothing for a formatting error; rate limits and timeouts are already retried in LitellmModel._generate, a layer down. - A model that answers a correction with a byte-identical reply is abandoned at once rather than burning the remaining attempts on a proven dead end. Only once the model's own attempts are spent does the parser guess, by escaping bare quotes inside string values with a string-aware scan. That guess is fallible and cannot be made otherwise -- a content quote followed by a comma is indistinguishable from a delimiter, and guessing wrong can silently shorten a value: {"a": "say "x", "b": "y"} -> {'a': 'say "x', 'b': 'y'} which parses cleanly with the tail of 'a' gone. In a guard spec a truncated rule reads as well-formed while encoding half a rule, so the salvage runs last and logs a warning naming the repair count. Candidate extraction is now brace-balanced and string-aware, so a brace inside a string value no longer ends the object early, and an unfenced reply stops at its own closing brace rather than swallowing trailing prose. Co-Authored-By: Claude Opus 5 (1M context) --- src/toolguard/buildtime/llm/llm_base.py | 254 ++++++++++++++++++---- tests/buildtime/llm/test_json_recovery.py | 182 ++++++++++++++++ tests/buildtime/llm/test_litellm_mock.py | 50 +++-- 3 files changed, 424 insertions(+), 62 deletions(-) create mode 100644 tests/buildtime/llm/test_json_recovery.py diff --git a/src/toolguard/buildtime/llm/llm_base.py b/src/toolguard/buildtime/llm/llm_base.py index f531ea1..b4eebaa 100644 --- a/src/toolguard/buildtime/llm/llm_base.py +++ b/src/toolguard/buildtime/llm/llm_base.py @@ -1,54 +1,226 @@ -import asyncio import json import re from abc import ABC -from typing import Dict, List +from typing import Dict, List, Optional, Tuple from loguru import logger from .i_tg_llm import I_TG_LLM +# Non-greedy: a reply that shows an example block before its real answer must +# yield the first block, not a fusion of the two. +_FENCED = re.compile(r"```(?:json)?\s*(\{.*?\})\s*```", re.DOTALL) + +_REPAIR_INSTRUCTION = ( + "Your previous reply could not be parsed as JSON. The parser reported:\n" + "{error}\n\n" + "Return the same content as a single valid JSON object and nothing else. " + 'Every double quote inside a string value must be escaped as \\" -- this ' + "applies to every field, not just some of them. Do not drop or summarize " + "any content, and do not add commentary before or after the object." +) + +# A `"` inside a string ends it only if the next non-space character is one of +# these (or the text runs out). Anything else means the quote was content. +_STRING_ENDERS = frozenset(",}]:") + class LanguageModelBase(I_TG_LLM, ABC): - async def chat_json( - self, messages: List[Dict], max_retries: int = 5, backoff_factor: float = 1.5 - ) -> Dict: - retries = 0 - while retries < max_retries: - response = await self.generate(messages) - res = self.extract_json_from_string(response) - if res is None: - wait_time = backoff_factor**retries + async def chat_json(self, messages: List[Dict], max_retries: int = 5) -> Dict: + """Ask for JSON, and on a parse failure tell the model what broke. + + The retry is a repair turn, not a replay: the unparseable reply and the + parser's own complaint go back to the model. Re-sending the original + prompt unchanged is what made a single quoted literal in a policy + document cost every attempt -- the model has no new information, so it + reproduces the same mistake five times. + """ + attempt_messages = list(messages) + previous_response: Optional[str] = None + last_response = "" + attempts_made = 0 + + for attempt in range(1, max_retries + 1): + last_response = await self.generate(attempt_messages) + attempts_made = attempt + # Salvage is a guess, so spend the model's own attempts first. + parsed, error = self.parse_json_response(last_response, allow_salvage=False) + if parsed is not None: + return parsed + + if last_response == previous_response: + # The model has ignored the same correction twice. Further + # attempts cost tokens and buy nothing. logger.warning( - f"Error: not json format. Retrying in {wait_time:.1f} seconds... (attempt {retries + 1}/{max_retries})" + "Model repeated an identical unparseable reply; abandoning " + "after {} attempt(s)", + attempt, ) - await asyncio.sleep(wait_time) - retries += 1 - else: - return res - raise RuntimeError("Exceeded maximum retries due to invalid JSON format.") - - def extract_json_from_string(self, s): - # Use regex to extract the JSON part from the string - match = re.search(r"```json\s*(\{.*?\})\s*```", s, re.DOTALL) - if match: - json_str = match.group(1) + break + + logger.warning( + "Response was not valid JSON ({}). Asking the model to fix it " + "(attempt {}/{})", + error, + attempt, + max_retries, + ) + attempt_messages = [ + *messages, + {"role": "assistant", "content": last_response}, + { + "role": "user", + "content": _REPAIR_INSTRUCTION.format(error=error or "unknown"), + }, + ] + previous_response = last_response + + # Out of model attempts. Guess, loudly, rather than lose the content. + parsed, error = self.parse_json_response(last_response, allow_salvage=True) + if parsed is not None: + return parsed + + raise RuntimeError( + f"Could not obtain valid JSON from the model after {attempts_made} " + f"attempt(s): {error}" + ) + + def parse_json_response( + self, s: str, allow_salvage: bool = True + ) -> Tuple[Optional[Dict], Optional[str]]: + """Parse a model reply, returning ``(object, error)``. + + Exactly one side is populated. The error text is the parser's own + message, so it can be handed back to the model as feedback. + + With ``allow_salvage``, a structurally complete reply whose only defect + is a bare double quote inside a string value is repaired locally. + """ + candidate = _json_candidate(s) + if candidate is None: + return None, "no JSON object found in the response" + + try: + return json.loads(candidate), None + except json.JSONDecodeError as exc: + strict_error = str(exc) + + if not allow_salvage: + return None, strict_error + + # Last resort. The model produced something structurally complete but + # left a double quote bare inside a string value -- the defect that + # shows up whenever a policy document quotes a literal. Escaping those + # quotes is a guess: a bare quote followed by a comma is + # indistinguishable from a delimiter, and guessing wrong can silently + # shorten a string value. Hence last, and hence loud. + repaired, repairs = _escape_bare_quotes(candidate) + if repairs: try: - return json.loads(json_str) - except json.JSONDecodeError as e: - logger.warning(f"Failed to decode JSON: {e}") - return None - else: - # Fallback: try to extract any JSON object from the string - match = re.search(r"(\{[\s\S]*\})", s) - if match: - json_str = match.group(1) - try: - return json.loads(json_str) - except json.JSONDecodeError as e: - logger.warning(f"Failed to parse JSON: {e}") - return None - - logger.debug("No JSON found in the string.") - logger.debug(s) - return None + parsed = json.loads(repaired) + except json.JSONDecodeError: + pass + else: + logger.warning( + "Salvaged an unparseable response by escaping {} bare quote(s) " + "inside string values. The affected values may be truncated " + "-- review them.", + repairs, + ) + return parsed, None + + return None, strict_error + + def extract_json_from_string(self, s: str) -> Optional[Dict]: + """The object, or ``None`` if nothing could be recovered.""" + parsed, _ = self.parse_json_response(s) + return parsed + + +def _json_candidate(s: str) -> Optional[str]: + """Return the substring most likely to be the JSON object. + + Prefers a fenced block, then falls back to the outermost brace-balanced + span. The balance scan is string-aware so a brace inside a string value + does not end the object early. + """ + if not s: + return None + + match = _FENCED.search(s) + if match: + return match.group(1).strip() + + start = s.find("{") + if start == -1: + return None + + depth = 0 + in_string = False + i = start + while i < len(s): + ch = s[i] + if in_string: + if ch == "\\": + i += 2 + continue + if ch == '"': + in_string = False + elif ch == '"': + in_string = True + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return s[start : i + 1] + i += 1 + + # Unbalanced -- hand back everything from the first brace so the parser + # produces a real error message for the repair turn. + return s[start:] + + +def _escape_bare_quotes(s: str) -> Tuple[str, int]: + """Escape double quotes that appear inside a JSON string value. + + Returns the rewritten text and the number of quotes escaped. Heuristic by + nature -- see the caller for why it is a last resort. + """ + out: List[str] = [] + i = 0 + n = len(s) + in_string = False + repairs = 0 + + while i < n: + ch = s[i] + if not in_string: + out.append(ch) + if ch == '"': + in_string = True + i += 1 + continue + + if ch == "\\": + out.append(s[i : i + 2]) + i += 2 + continue + + if ch == '"': + j = i + 1 + while j < n and s[j] in " \t\r\n": + j += 1 + if j >= n or s[j] in _STRING_ENDERS: + out.append(ch) + in_string = False + else: + out.append('\\"') + repairs += 1 + i += 1 + continue + + out.append(ch) + i += 1 + + return "".join(out), repairs diff --git a/tests/buildtime/llm/test_json_recovery.py b/tests/buildtime/llm/test_json_recovery.py new file mode 100644 index 0000000..f99ade4 --- /dev/null +++ b/tests/buildtime/llm/test_json_recovery.py @@ -0,0 +1,182 @@ +"""Recovery from the one JSON defect models actually make: bare inner quotes. + +A policy document that quotes a literal — ``obtain the user's explicit +confirmation ("yes")`` — makes models escape the quotes in one field and leave +them bare in another, in the same reply. The payload is complete and +brace-balanced; only the parse fails. These tests pin the two layers that get +the content back: a retry that tells the model what broke, and a last-resort +salvage for a model that will not cooperate. +""" + +import json +from typing import Dict, List + +import pytest + +from toolguard.buildtime.llm.llm_base import LanguageModelBase + +# Escaped inside "references", bare inside "description" -- the reported shape. +BARE_QUOTES_PAYLOAD = """```json +{ + "tool_name": "delete_record", + "policy_items": [ + { + "description": "The agent must first list the details of the action and obtain the user's explicit confirmation ("yes") before proceeding.", + "references": ["obtain the user's explicit confirmation (\\"yes\\")"] + } + ] +} +```""" + +# A payload whose bare quote is followed by a comma, which the salvage cannot +# tell from a delimiter -- used to exercise the retry path rather than salvage. +BARE_QUOTES_PAYLOAD_UNSALVAGEABLE = """```json +{"note": "he said "hello", then left", "n": 3} +```""" + +GOOD_PAYLOAD = """```json +{"tool_name": "delete_record", "policy_items": []} +```""" + + +class ScriptedModel(LanguageModelBase): + """Returns queued replies and records the messages it was asked with.""" + + def __init__(self, replies: List[str]): + self.replies = list(replies) + self.seen: List[List[Dict]] = [] + + async def generate(self, messages: List[Dict]) -> str: + self.seen.append([dict(m) for m in messages]) + return self.replies.pop(0) if self.replies else "not json at all" + + +class AlwaysBareQuotes(LanguageModelBase): + """Reproduces the defect however it is corrected, rewording each time.""" + + def __init__(self): + self.call_count = 0 + + async def generate(self, messages: List[Dict]) -> str: + self.call_count += 1 + return BARE_QUOTES_PAYLOAD.replace( + "delete_record", f"delete_record_{self.call_count}" + ) + + +# --- the salvage --------------------------------------------------------- + + +def test_salvage_recovers_a_payload_with_bare_inner_quotes(): + model = ScriptedModel([]) + + result = model.extract_json_from_string(BARE_QUOTES_PAYLOAD) + + assert result is not None + item = result["policy_items"][0] + assert item["description"].endswith('("yes") before proceeding.') + assert item["references"] == ['obtain the user\'s explicit confirmation ("yes")'] + + +@pytest.mark.parametrize( + "payload", + [ + '{"a": "b", "c": [1, 2], "d": {"e": "f"}}', + '{"s": "comma, inside", "t": "colon: inside", "u": "brace } inside"}', + '{"esc": "already \\"escaped\\" fine"}', + '{"empty": "", "next": "v"}', + '{"bracket": "an [array] inside", "n": null}', + ], +) +def test_salvage_leaves_valid_payloads_untouched(payload): + model = ScriptedModel([]) + + assert model.extract_json_from_string(payload) == json.loads(payload) + + +def test_the_first_fenced_block_wins(): + """A model that shows an example before its answer must not fuse the two.""" + model = ScriptedModel([]) + + text = ( + 'For example:\n```json\n{"shape": "example"}\n```\n' + 'And here is the answer:\n```json\n{"shape": "answer"}\n```' + ) + + assert model.extract_json_from_string(text) == {"shape": "example"} + + +def test_an_unfenced_object_stops_at_its_own_closing_brace(): + model = ScriptedModel([]) + + text = 'Here you go: {"a": {"b": "}"}} -- hope that helps!' + + assert model.extract_json_from_string(text) == {"a": {"b": "}"}} + + +@pytest.mark.parametrize( + "payload", + [ + "This is not JSON", + '{"key": "value"', # truncated + "", + ], +) +def test_salvage_does_not_invent_a_result(payload): + model = ScriptedModel([]) + + assert model.extract_json_from_string(payload) is None + + +# --- the repair turn ---------------------------------------------------- + + +async def test_retry_tells_the_model_what_broke(): + model = ScriptedModel([BARE_QUOTES_PAYLOAD_UNSALVAGEABLE, GOOD_PAYLOAD]) + + result = await model.chat_json([{"role": "user", "content": "generate a spec"}]) + + assert result == {"tool_name": "delete_record", "policy_items": []} + + # The second attempt must carry the failed reply plus the parser's complaint, + # otherwise the model has nothing new to work with and repeats itself. + second_attempt = model.seen[1] + assert len(second_attempt) > 1 + assert any( + m["role"] == "assistant" and BARE_QUOTES_PAYLOAD_UNSALVAGEABLE in m["content"] + for m in second_attempt + ) + feedback = second_attempt[-1] + assert feedback["role"] == "user" + assert "JSON" in feedback["content"] + + +async def test_retry_does_not_resend_an_unchanged_prompt(): + model = ScriptedModel([]) # every reply is unparseable + + with pytest.raises(RuntimeError): + await model.chat_json( + [{"role": "user", "content": "generate a spec"}], max_retries=3 + ) + + prompts = [json.dumps(msgs) for msgs in model.seen] + assert len(prompts) == len(set(prompts)), "identical prompts were re-sent" + + +async def test_unrecoverable_response_still_raises_after_retries(): + model = ScriptedModel([]) + + with pytest.raises(RuntimeError, match="valid JSON"): + await model.chat_json([{"role": "user", "content": "hi"}], max_retries=2) + + +async def test_salvage_catches_a_model_that_will_not_fix_itself(): + """Salvage is a guess, so the model gets its own chances first.""" + model = AlwaysBareQuotes() + + result = await model.chat_json([{"role": "user", "content": "hi"}], max_retries=3) + + assert result["policy_items"][0]["description"].endswith( + '("yes") before proceeding.' + ) + assert model.call_count == 3, "salvage must not pre-empt the repair turns" diff --git a/tests/buildtime/llm/test_litellm_mock.py b/tests/buildtime/llm/test_litellm_mock.py index dd18e3e..d56bc89 100644 --- a/tests/buildtime/llm/test_litellm_mock.py +++ b/tests/buildtime/llm/test_litellm_mock.py @@ -238,37 +238,45 @@ async def test_chat_json_retry_on_invalid_json(mock_model): json_data = {"valid": "json"} with patch("toolguard.buildtime.llm.tg_litellm.acompletion") as mock_acompletion: - with patch("toolguard.buildtime.llm.tg_litellm.asyncio.sleep") as mock_sleep: - # First two responses are invalid, third is valid - mock_acompletion.side_effect = [ - create_mock_response("This is not JSON"), - create_mock_response("Still not JSON"), - create_mock_response(json.dumps(json_data)), - ] + # First two responses are invalid, third is valid + mock_acompletion.side_effect = [ + create_mock_response("This is not JSON"), + create_mock_response("Still not JSON"), + create_mock_response(json.dumps(json_data)), + ] + + messages = [{"role": "user", "content": "Give me JSON"}] + result = await mock_model.chat_json(messages) - messages = [{"role": "user", "content": "Give me JSON"}] - result = await mock_model.chat_json(messages) + assert result == json_data + assert mock_acompletion.call_count == 3 - assert result == json_data - assert mock_acompletion.call_count == 3 - assert mock_sleep.call_count == 2 + # Each retry is a repair turn: the rejected reply and the reason go + # back to the model, rather than the original prompt being replayed. + retry_messages = mock_acompletion.call_args_list[1].kwargs["messages"] + assert retry_messages[-2] == { + "role": "assistant", + "content": "This is not JSON", + } + assert "JSON" in retry_messages[-1]["content"] @pytest.mark.asyncio async def test_chat_json_max_retries_exceeded(mock_model): - """Test that RuntimeError is raised after max retries for invalid JSON.""" + """Test that RuntimeError is raised once the model stops making progress.""" with patch("toolguard.buildtime.llm.tg_litellm.acompletion") as mock_acompletion: - with patch("toolguard.buildtime.llm.tg_litellm.asyncio.sleep"): - # Always return invalid JSON - mock_acompletion.return_value = create_mock_response("Not JSON at all") + # Always return the same invalid JSON + mock_acompletion.return_value = create_mock_response("Not JSON at all") - messages = [{"role": "user", "content": "Give me JSON"}] + messages = [{"role": "user", "content": "Give me JSON"}] - with pytest.raises(RuntimeError) as exc_info: - await mock_model.chat_json(messages) + with pytest.raises(RuntimeError) as exc_info: + await mock_model.chat_json(messages) - assert "Exceeded maximum retries" in str(exc_info.value) - assert mock_acompletion.call_count == 5 + assert "valid JSON" in str(exc_info.value) + # A model that answers a correction with the identical reply will not + # be talked round; spending the remaining attempts on it is waste. + assert mock_acompletion.call_count == 2 @pytest.mark.asyncio From f9bb61c919c336e78a84fcb5807a98e8f1b9754c Mon Sep 17 00:00:00 2001 From: naamaz Date: Thu, 13 Aug 2026 11:39:57 +0300 Subject: [PATCH 5/6] Make a partial spec set an error that names the tools it lost A run that quietly produces 31 specs for 33 tools ships two tools unguarded, and the shortfall is indistinguishable from a smaller request: we found the two missing tools by counting files in the output directory. Add SpecGenerationError, shared by both generators. It carries both sides of the outcome -- the specs that generated cleanly and were written, and the ToolFailure entries naming those that did not -- so one except clause works across v1 and v2. v1 fanned out with a bare asyncio.gather, so the first exception discarded every other tool's finished spec: 33 tools lost to one unparseable response. Isolate each tool, then raise once at the end. v2 already isolated per tool, but recorded the loss only in a logger.error and returned a short list. Its ToolErrorPolicy enum was also declared and never wired to the field, which was a bare str -- so on_tool_error="skipp" silently meant "raise". Wire the enum up and add a third policy: skip return the partial list, as before raise abort at the first failure raise_at_end finish and write every other tool, then raise (new default) raise_at_end is the default because a missing guard spec is a security outcome, not a logging detail. Nothing that succeeded is lost -- the files are written before the raise -- so a failure still costs only its own tool. This changes public behaviour: callers relying on a partial list must pass on_tool_error="skip". test_a_failing_tool_does_not_abort_the_others tracked the old default and now asserts isolation via the exception's .specs. The field uses Field(default=...) rather than a bare member assignment because the pre-commit mypy hook runs without the project installed, so it cannot resolve the StrEnum compat shim and reads the member as a plain str. Co-Authored-By: Claude Opus 5 (1M context) --- src/toolguard/buildtime/__init__.py | 6 ++ src/toolguard/buildtime/buildtime.py | 6 ++ src/toolguard/buildtime/gen_spec/errors.py | 37 +++++++ .../buildtime/gen_spec/spec_generator.py | 39 +++++-- .../buildtime/gen_spec_v2/__init__.py | 6 ++ .../buildtime/gen_spec_v2/pipeline.py | 34 +++++- .../gen_spec/test_spec_generator_isolation.py | 100 ++++++++++++++++++ tests/buildtime/gen_spec_v2/test_pipeline.py | 62 +++++++++-- 8 files changed, 268 insertions(+), 22 deletions(-) create mode 100644 src/toolguard/buildtime/gen_spec/errors.py create mode 100644 tests/buildtime/gen_spec/test_spec_generator_isolation.py diff --git a/src/toolguard/buildtime/__init__.py b/src/toolguard/buildtime/__init__.py index 88d4eff..7363bcb 100644 --- a/src/toolguard/buildtime/__init__.py +++ b/src/toolguard/buildtime/__init__.py @@ -11,9 +11,11 @@ PolicySpecStep, ) from toolguard.buildtime.gen_spec.data_types import ToolInfo +from toolguard.buildtime.gen_spec.errors import SpecGenerationError, ToolFailure from toolguard.buildtime.gen_spec_v2 import ( SpecV2, SpecV2Options, + ToolErrorPolicy, generate_guard_examples_v2, generate_guard_specs_v2, generate_guard_specs_v2_full, @@ -35,6 +37,10 @@ "ToolInfo", "PolicySpecOptions", "PolicySpecStep", + # partial-run reporting, shared by both generators + "SpecGenerationError", + "ToolFailure", + "ToolErrorPolicy", # v2 spec generation (alternative to generate_guard_specs) "generate_guard_specs_v2", "generate_spec_conflicts_v2", diff --git a/src/toolguard/buildtime/buildtime.py b/src/toolguard/buildtime/buildtime.py index 7f339f4..f3d995d 100644 --- a/src/toolguard/buildtime/buildtime.py +++ b/src/toolguard/buildtime/buildtime.py @@ -50,6 +50,12 @@ async def generate_guard_specs( Returns: List of ToolGuardSpec objects containing the generated specifications. + + Raises: + SpecGenerationError: if any tool failed. Every other tool still ran and + its spec is already written; the error names what was lost, so a + short spec set cannot pass for a small request. The successful + specs are on the exception as ``.specs``. """ work_dir = Path(work_dir) work_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/toolguard/buildtime/gen_spec/errors.py b/src/toolguard/buildtime/gen_spec/errors.py new file mode 100644 index 0000000..2a6a150 --- /dev/null +++ b/src/toolguard/buildtime/gen_spec/errors.py @@ -0,0 +1,37 @@ +"""Failure reporting shared by both spec generators. + +A build that quietly produces 31 specs for 33 tools ships two tools unguarded, +and the shortfall is indistinguishable from a smaller request. Both generators +therefore finish every healthy tool, write it, and then raise one error that +names what was lost -- so a caller learns which tools failed without counting +files in the output directory. +""" + +from typing import Any, List, Sequence + +from pydantic import BaseModel + + +class ToolFailure(BaseModel): + """One tool's generation failure.""" + + tool_name: str + error: str + + +class SpecGenerationError(Exception): + """Some tools failed. Carries both sides of the outcome. + + ``specs`` are the specs that generated cleanly and were written to disk -- + v1's ``ToolGuardSpec`` or v2's ``SpecV2``, depending on which generator + raised. ``failures`` name the tools that did not. + """ + + def __init__(self, failures: Sequence[ToolFailure], specs: Sequence[Any]): + self.failures: List[ToolFailure] = list(failures) + self.specs: List[Any] = list(specs) + detail = "; ".join(f"{f.tool_name}: {f.error}" for f in self.failures) + super().__init__( + f"Spec generation failed for {len(self.failures)} tool(s) " + f"({len(self.specs)} succeeded): {detail}" + ) diff --git a/src/toolguard/buildtime/gen_spec/spec_generator.py b/src/toolguard/buildtime/gen_spec/spec_generator.py index 8df11f9..e33ee68 100644 --- a/src/toolguard/buildtime/gen_spec/spec_generator.py +++ b/src/toolguard/buildtime/gen_spec/spec_generator.py @@ -8,6 +8,7 @@ from toolguard.buildtime.compat.strenum import StrEnum from toolguard.buildtime.data_types import TOOLS from toolguard.buildtime.gen_spec.data_types import ToolInfo +from toolguard.buildtime.gen_spec.errors import SpecGenerationError, ToolFailure from toolguard.buildtime.gen_spec.fn_to_toolinfo import function_to_toolInfo from toolguard.buildtime.gen_spec.oas_to_toolinfo import openapi_to_toolinfos from toolguard.buildtime.gen_spec.utils import ( @@ -102,20 +103,40 @@ async def extract_toolguard_specs( llm, policy_text, tool_infos, process_dir, options ) - async def do_one_tool(tool_name: str) -> ToolGuardSpec: - spec = await generator.generate_policy(tool_name) + failures: List[ToolFailure] = [] + + async def do_one_tool(tool_name: str) -> Optional[ToolGuardSpec]: + # Isolated per tool: one tool whose response will not parse used to + # discard every other tool's finished spec in the same call. + try: + spec = await generator.generate_policy(tool_name) + except Exception as ex: # noqa: BLE001 - per-tool isolation by design + logger.error("Spec generation failed for '{}': {}", tool_name, ex) + failures.append(ToolFailure(tool_name=tool_name, error=str(ex))) + return None if spec.policy_items: save_output(step1_output_dir, tool_name + ".json", spec) return spec - specs = await asyncio.gather( - *[ - do_one_tool(tool.name) - for tool in tool_infos - if ((tools2guard is None) or (tool.name in tools2guard)) - ] - ) + targets = [ + tool.name + for tool in tool_infos + if ((tools2guard is None) or (tool.name in tools2guard)) + ] + results = await asyncio.gather(*[do_one_tool(name) for name in targets]) logger.debug("All tools done") + + specs = [spec for spec in results if spec is not None] + + # gather() completes out of order; report in the order the tools were asked for. + order = {name: i for i, name in enumerate(targets)} + failures.sort(key=lambda f: order[f.tool_name]) + + if failures: + # Everything that worked is already on disk, so raising here loses + # nothing and stops a short spec set from passing for a small request. + raise SpecGenerationError(failures, specs) + return specs diff --git a/src/toolguard/buildtime/gen_spec_v2/__init__.py b/src/toolguard/buildtime/gen_spec_v2/__init__.py index 80c505c..83ceaf0 100644 --- a/src/toolguard/buildtime/gen_spec_v2/__init__.py +++ b/src/toolguard/buildtime/gen_spec_v2/__init__.py @@ -38,7 +38,10 @@ Trigger, ) from toolguard.buildtime.gen_spec_v2.pipeline import ( + SpecGenerationError, SpecV2Options, + ToolErrorPolicy, + ToolFailure, generate_guard_examples_v2, generate_guard_specs_v2, generate_guard_specs_v2_full, @@ -59,6 +62,9 @@ "generate_guard_specs_v2_full", "generate_guard_examples_v2", "SpecV2Options", + "ToolErrorPolicy", + "SpecGenerationError", + "ToolFailure", # adapter to what codegen and the runtime consume "spec_v2_to_v1", "specs_v2_to_v1", diff --git a/src/toolguard/buildtime/gen_spec_v2/pipeline.py b/src/toolguard/buildtime/gen_spec_v2/pipeline.py index 0de93e8..a8c2f42 100644 --- a/src/toolguard/buildtime/gen_spec_v2/pipeline.py +++ b/src/toolguard/buildtime/gen_spec_v2/pipeline.py @@ -22,6 +22,7 @@ from toolguard.buildtime.compat.strenum import StrEnum from toolguard.buildtime.gen_spec.data_types import ToolInfo +from toolguard.buildtime.gen_spec.errors import SpecGenerationError, ToolFailure from toolguard.buildtime.gen_spec_v2.conflicts import attach_conflicts, find_conflicts from toolguard.buildtime.gen_spec_v2.context import GenContext from toolguard.buildtime.gen_spec_v2.models import SpecDebugV2, SpecV2 @@ -43,10 +44,18 @@ class ToolErrorPolicy(StrEnum): """What to do when one tool's generation raises.""" skip = "skip" - """Record the failure and let every other tool finish.""" + """Record the failure in the log and return the specs that did work.""" raise_ = "raise" - """Abort the whole run.""" + """Abort the whole run at the first failure.""" + + raise_at_end = "raise_at_end" + """Let every other tool finish and write, then raise naming the failures. + + The default. A guard spec that silently goes missing is a tool that ships + unguarded, so a partial set has to be an error the caller cannot overlook + -- while still costing only the tools that actually failed. + """ class SpecV2Options(BaseModel): @@ -61,7 +70,7 @@ class SpecV2Options(BaseModel): description="None = let the model choose, >0 = exactly that many per side", ) max_concurrency: int = Field(default=8, ge=1) - on_tool_error: str = "skip" + on_tool_error: ToolErrorPolicy = Field(default=ToolErrorPolicy.raise_at_end) async def _generate_one( @@ -110,6 +119,11 @@ async def generate_guard_specs_v2( Writes ``.json``. Note these are the same filenames v1 uses, and v1's loader will accept them while silently ignoring the v2 fields — point v2 at its own directory rather than sharing one with v1 output. + + Raises: + SpecGenerationError: under the default ``raise_at_end`` policy, once + every other tool has finished and been written. Pass + ``on_tool_error="skip"`` to get the partial list back instead. """ options = options or SpecV2Options() work_dir = Path(work_dir) @@ -127,15 +141,17 @@ async def generate_guard_specs_v2( targets = [tool for tool in ctx.tools if tool.name in set(tools2guard)] semaphore = asyncio.Semaphore(options.max_concurrency) + failures: List[ToolFailure] = [] async def guarded(tool: ToolInfo): async with semaphore: try: return await _generate_one(llm, ctx, tool, source_doc, options) except Exception as ex: # noqa: BLE001 - per-tool isolation by design - if options.on_tool_error == "raise": + if options.on_tool_error == ToolErrorPolicy.raise_: raise logger.error("Spec generation failed for '{}': {}", tool.name, ex) + failures.append(ToolFailure(tool_name=tool.name, error=str(ex))) return None results = await asyncio.gather(*[guarded(tool) for tool in targets]) @@ -145,6 +161,16 @@ async def guarded(tool: ToolInfo): dump_spec(spec, work_dir / f"{spec.tool_name}.json") logger.debug("gen_spec_v2: wrote {}/{} spec(s)", len(specs), len(targets)) + + # gather() completes out of order; report in the order the tools were asked for. + order = {tool.name: i for i, tool in enumerate(targets)} + failures.sort(key=lambda f: order[f.tool_name]) + + if failures and options.on_tool_error == ToolErrorPolicy.raise_at_end: + # Written first, then raised: nothing that succeeded is lost, and the + # shortfall cannot be mistaken for a smaller request. + raise SpecGenerationError(failures, specs) + return specs diff --git a/tests/buildtime/gen_spec/test_spec_generator_isolation.py b/tests/buildtime/gen_spec/test_spec_generator_isolation.py new file mode 100644 index 0000000..667011b --- /dev/null +++ b/tests/buildtime/gen_spec/test_spec_generator_isolation.py @@ -0,0 +1,100 @@ +"""One tool's failure must not take the rest of the run with it. + +v1 fanned out over the tools with a bare ``asyncio.gather``, so the first +exception discarded every other tool's finished spec -- 33 tools lost to one +unparseable response. +""" + +from pathlib import Path +from typing import Dict, List + +import pytest + +from toolguard.buildtime.gen_spec.spec_generator import ( + PolicySpecOptions, + SpecGenerationError, + extract_toolguard_specs, +) +from toolguard.buildtime.llm import I_TG_LLM + +POLICY = "Only HR may edit an employee's data." + + +def update_employee(user_id: int, salary: float) -> dict: + """Update an employee.""" + return {} + + +def get_employee(user_id: int) -> dict: + """Read an employee.""" + return {} + + +TOOLS = [update_employee, get_employee] + +SPEC_RESPONSE = { + "policy_items": [ + { + "name": "HR only", + "description": "Only HR may edit.", + "references": ["Only HR may edit an employee's data."], + } + ] +} + + +class BoomForOneTool(I_TG_LLM): + """Answers every prompt, except those naming the doomed tool.""" + + def __init__(self, doomed: str): + self.doomed = doomed + self.seen: List[str] = [] + + async def chat_json(self, messages: List[Dict]) -> Dict: + content = messages[-1]["content"] + self.seen.append(content) + # Key on the target section, not the catalog that every prompt carries, + # so only the doomed tool's pipeline fails. + _, _, target = content.partition("Target Tool:") + if f'"name": "{self.doomed}"' in target: + raise RuntimeError("could not obtain valid JSON") + return SPEC_RESPONSE + + async def generate(self, messages: List[Dict]) -> str: + raise NotImplementedError + + +def _options() -> PolicySpecOptions: + # Only the first step and no examples, so the test pins fan-out behaviour + # rather than the content of the later passes. + return PolicySpecOptions(spec_steps={"CREATE_POLICIES"}, example_number=0) + + +async def test_a_failing_tool_does_not_discard_the_others(tmp_path: Path): + llm = BoomForOneTool("get_employee") + + with pytest.raises(SpecGenerationError) as exc_info: + await extract_toolguard_specs(POLICY, TOOLS, tmp_path, llm, options=_options()) + + assert [s.tool_name for s in exc_info.value.specs] == ["update_employee"] + assert (tmp_path / "update_employee.json").exists() + + +async def test_the_error_names_the_failed_tool(tmp_path: Path): + llm = BoomForOneTool("get_employee") + + with pytest.raises(SpecGenerationError) as exc_info: + await extract_toolguard_specs(POLICY, TOOLS, tmp_path, llm, options=_options()) + + assert [f.tool_name for f in exc_info.value.failures] == ["get_employee"] + assert "get_employee" in str(exc_info.value) + + +async def test_a_clean_run_still_returns_every_spec(tmp_path: Path): + llm = BoomForOneTool("nothing_matches_this") + + specs = await extract_toolguard_specs( + POLICY, TOOLS, tmp_path, llm, options=_options() + ) + + assert sorted(s.tool_name for s in specs) == ["get_employee", "update_employee"] diff --git a/tests/buildtime/gen_spec_v2/test_pipeline.py b/tests/buildtime/gen_spec_v2/test_pipeline.py index 6ad6823..8053e87 100644 --- a/tests/buildtime/gen_spec_v2/test_pipeline.py +++ b/tests/buildtime/gen_spec_v2/test_pipeline.py @@ -3,9 +3,11 @@ import json import pytest +from pydantic import ValidationError from toolguard.buildtime.gen_spec_v2.models import PendingType, Trigger from toolguard.buildtime.gen_spec_v2.pipeline import ( + SpecGenerationError, SpecV2Options, generate_guard_specs_v2, generate_guard_specs_v2_full, @@ -179,22 +181,64 @@ async def test_pending_gaps_reach_the_written_spec(tmp_path): assert pending.suggested_source == "system_vars:blacklist" +class _BoomFor(FakeLLM): + """Fails the pipeline of one named tool and no other.""" + + def __init__(self, responses, tool_name): + super().__init__(responses) + self.tool_name = tool_name + + async def chat_json(self, messages): + # Key on the tool under generation, not the catalog every prompt + # carries, so only the named tool's pipeline fails. + if f"Tool name: {self.tool_name}" in messages[-1]["content"]: + raise RuntimeError("model exploded") + return await super().chat_json(messages) + + async def test_a_failing_tool_does_not_abort_the_others(tmp_path): - class Boom(FakeLLM): - async def chat_json(self, messages): - # Key on the tool under generation, not the catalog every prompt - # carries, so only get_employee's pipeline fails. - if "Tool name: get_employee" in messages[-1]["content"]: - raise RuntimeError("model exploded") - return await super().chat_json(messages) + llm = _BoomFor(_responses(), "get_employee") - llm = Boom(_responses()) + with pytest.raises(SpecGenerationError) as exc_info: + await generate_guard_specs_v2(POLICY, TOOLS, llm, tmp_path) + + # Every healthy tool still ran, and its spec still reached disk. + assert [s.tool_name for s in exc_info.value.specs] == ["update_employee"] + assert (tmp_path / "update_employee.json").exists() + + +async def test_the_error_names_the_tools_that_failed(tmp_path): + llm = _BoomFor(_responses(), "get_employee") - specs = await generate_guard_specs_v2(POLICY, TOOLS, llm, tmp_path) + with pytest.raises(SpecGenerationError) as exc_info: + await generate_guard_specs_v2(POLICY, TOOLS, llm, tmp_path) + + error = exc_info.value + assert [f.tool_name for f in error.failures] == ["get_employee"] + assert "model exploded" in error.failures[0].error + # A caller that only logs the exception must still learn which tool it was. + assert "get_employee" in str(error) + + +async def test_skip_keeps_the_old_partial_result_behaviour(tmp_path): + llm = _BoomFor(_responses(), "get_employee") + + specs = await generate_guard_specs_v2( + POLICY, + TOOLS, + llm, + tmp_path, + options=SpecV2Options(on_tool_error="skip"), + ) assert [s.tool_name for s in specs] == ["update_employee"] +async def test_a_misspelled_error_policy_is_rejected(tmp_path): + with pytest.raises(ValidationError): + SpecV2Options(on_tool_error="skipp") + + async def test_on_tool_error_raise_propagates(tmp_path): class Boom(FakeLLM): async def chat_json(self, messages): From 93bcbe200ef4db145b43dce02b4b8376349b2d13 Mon Sep 17 00:00:00 2001 From: naamaz Date: Thu, 13 Aug 2026 11:40:22 +0300 Subject: [PATCH 6/6] Fix the e2e imports so three test modules collect again test_calculator, test_gen_spec_v2_codegen and test_guard_set_delta all failed to collect with "No module named 'examples'". The package is not missing -- it is tests/examples -- but they imported it as a top-level name, which only works with tests/ itself on sys.path. It is not: tests/ has an __init__.py, so pytest walks past it and inserts the repo root, which makes tests.examples importable but not examples. Import them as tests.examples.calculator.inputs instead of adding tests/ to sys.path, which would also expose buildtime, runtime, data and tmp as top-level names and leave a shadowing hazard for anyone later writing "import runtime". Add the empty tests/examples/__init__.py that was the only gap in the chain, so the import resolves through regular packages rather than a PEP 420 namespace portion wedged between two of them. Collection goes from 357 tests with 3 errors to 367 with none. Note this un-hides test_guard_set_delta, which is marked @pytest.mark.delta and documented in pyproject as opt-in ("two full LLM spec runs"). Nothing deselects it -- addopts has no -m filter -- so it only stayed unrun because this import was broken, and a plain "pytest tests" now includes it. Co-Authored-By: Claude Opus 5 (1M context) --- tests/buildtime/e2e/test_calculator.py | 6 +++--- tests/buildtime/e2e/test_gen_spec_v2_codegen.py | 6 +++--- tests/buildtime/e2e/test_guard_set_delta.py | 2 +- tests/examples/__init__.py | 0 4 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 tests/examples/__init__.py diff --git a/tests/buildtime/e2e/test_calculator.py b/tests/buildtime/e2e/test_calculator.py index f1be7a0..039ed8d 100644 --- a/tests/buildtime/e2e/test_calculator.py +++ b/tests/buildtime/e2e/test_calculator.py @@ -6,9 +6,9 @@ import markdown # type: ignore[import] import pytest -from examples.calculator.inputs import tool_functions as fn_tools -from examples.calculator.inputs import tool_langchain as lg_tools -from examples.calculator.inputs import tool_methods as mtd_tools +from tests.examples.calculator.inputs import tool_functions as fn_tools +from tests.examples.calculator.inputs import tool_langchain as lg_tools +from tests.examples.calculator.inputs import tool_methods as mtd_tools from toolguard.buildtime.gen_spec.spec_generator import ( PolicySpecOptions, PolicySpecStep, diff --git a/tests/buildtime/e2e/test_gen_spec_v2_codegen.py b/tests/buildtime/e2e/test_gen_spec_v2_codegen.py index 16e7c68..04ccc90 100644 --- a/tests/buildtime/e2e/test_gen_spec_v2_codegen.py +++ b/tests/buildtime/e2e/test_gen_spec_v2_codegen.py @@ -20,9 +20,9 @@ import pytest from dotenv import load_dotenv -from examples.calculator.inputs import tool_functions as fn_tools -from examples.calculator.inputs import tool_langchain as lg_tools -from examples.calculator.inputs import tool_methods as mtd_tools +from tests.examples.calculator.inputs import tool_functions as fn_tools +from tests.examples.calculator.inputs import tool_langchain as lg_tools +from tests.examples.calculator.inputs import tool_methods as mtd_tools from toolguard.buildtime import ( LitellmModel, diff --git a/tests/buildtime/e2e/test_guard_set_delta.py b/tests/buildtime/e2e/test_guard_set_delta.py index 4bc12ed..f63a584 100644 --- a/tests/buildtime/e2e/test_guard_set_delta.py +++ b/tests/buildtime/e2e/test_guard_set_delta.py @@ -24,7 +24,7 @@ import pytest from dotenv import load_dotenv -from examples.calculator.inputs import tool_functions as fn_tools +from tests.examples.calculator.inputs import tool_functions as fn_tools from toolguard.buildtime import ( LitellmModel, diff --git a/tests/examples/__init__.py b/tests/examples/__init__.py new file mode 100644 index 0000000..e69de29