diff --git a/.abcd/development/brief/04-surfaces/31-lab.md b/.abcd/development/brief/04-surfaces/31-lab.md index 7ce8b9019b..6068c77a9b 100644 --- a/.abcd/development/brief/04-surfaces/31-lab.md +++ b/.abcd/development/brief/04-surfaces/31-lab.md @@ -128,6 +128,31 @@ Coverage grades each section grounded, partial or blank, the lifeboat's vocabulary. The harvest files nothing, and claims no cost the lab did not measure. +A lab that collected the decision adapter's shadow pairs +(itd-2609221009495079) gets one more section. When a judgement type is +configured for shadow (`oracle.judgements.` set to +`shadow:/`, [`03-configuration.md`](../05-internals/03-configuration.md)), +the host decides and the adapter answers beside it. Each pair is appended to +`pairs.jsonl` in the newest lab home minted for the repository: the host's +answer, the adapter's answer, the adapter's probability, the state's SHA-256 and +size, and the model that answered, or why it did not. That is the only file the +adapter writes, and it is never written into the repository. The drain's +judgement before a lane opens (`eligibility-residual`) is today the one step +that goes through the router, so it is the one type whose pairs are kept; +every other type stays on the host whatever its route says +([`03-configuration.md`](../05-internals/03-configuration.md)). +Here the harvest +computes something rather than only assembling prose: from `pairs.jsonl` it +counts, for each judgement type, the pairs, the pairs compared (the adapter +answered), the pairs that agreed and the pairs skipped (the state was larger +than the model's window, or the call failed). It reports the agreement rate, +and the compared and agreed counts in four probability bands (below 0.50, +0.50–0.70, 0.70–0.90, 0.90 and above). Lines that do not parse are counted +apart and never guessed at. The section ends with the line a research note +filed from it carries, naming the lab home and the count. That note is the +evidence a ruling turns a type on with. A lab with no pairs has no such +section. + ## Halt and record A gate refusal halts the lab and is recorded, never adapted around. When the diff --git a/.abcd/development/brief/05-internals/02-adapters.md b/.abcd/development/brief/05-internals/02-adapters.md index 4c0ab7a55e..fb056b9f1b 100644 --- a/.abcd/development/brief/05-internals/02-adapters.md +++ b/.abcd/development/brief/05-internals/02-adapters.md @@ -247,6 +247,45 @@ is sent to a provider. No test reaches a real provider: the client and every dispatching verb are exercised end to end against a fake on the loopback address, and the key is built at run time. +### The decision adapter — typed judgements, measured in a lab first + +The decision adapter (itd-2609221009495079, `core/decide/judge`) answers abcd's +closed-option judgements with a value and a probability. It has no generative +step. Every judgement goes through one interface, `(state, typed question) → +(value, probability)`. A question is one of three kinds: choose one of N, +score on a scale, or yes/no. Its type is one of a closed registry +([`03-configuration.md`](03-configuration.md) lists it). The host's own +judgement implements the interface at probability 1, and so does the decision +model reached through the API adapter above, so the two are compared like for +like. The adapter renders the same prompt the host is shown, sends it through +`APIConfig.Call` (which admits the target against the allowlist and the +denylist again and resolves the key by name), and reads back one JSON object +with a `value` among the question's choices and a `probability` from 0 to 1. +The model cannot abstain, so any other answer is not an answer. + +A router switches each type by configuration alone, so no caller changes when +a type moves. With `host`, or nothing set, the host decides. With +`shadow:/`, the host decides and the adapter's answer is +recorded beside it, as a pair in the newest lab home's `pairs.jsonl`, never in +the repository. A state larger than the model's 32K window is recorded as +skipped with its size. With a bare `/`, the adapter decides, +and the run record names the route and the model the provider reported; only +the person's own `~/.abcd.noindex/config.json` turns a type on, and the same +route in a repository's file is skipped so the machine's route, or the host, +applies ([`03-configuration.md`](03-configuration.md)). An +adapter that cannot answer leaves the decision to the host, with a fallback +receipt saying why. The allowlist refusal of adr-2609221009491186 happens when +the configuration is read, before any call. `abcd lab harvest` turns the pairs +into agreement per type, with counts and probability bands, and the research +note filed from it is the evidence a ruling turns a type on with. The first +candidate is `typesafe/jev-1.13` on OpenRouter's list. The live paid check +against it is owed to a person (iss-2609291925398978). The drain's judgement +(`eligibility-residual`) is the one type whose step calls the router: the +drain's move asks it in each mode. No other type's step calls the router yet, +so every other judgement stays on the host whatever its route says. The tests +drive the router, the pairs file, the harvest and the drain's moves in each +mode with a fake adapter and a temporary home. + ### Reviewer routes — the command-line runner, and RepoPrompt over MCP A build lane's ruthless and security reviews, which the validate stage hands diff --git a/.abcd/development/brief/05-internals/03-configuration.md b/.abcd/development/brief/05-internals/03-configuration.md index aa7b383581..0da0b52593 100644 --- a/.abcd/development/brief/05-internals/03-configuration.md +++ b/.abcd/development/brief/05-internals/03-configuration.md @@ -210,7 +210,8 @@ rather than skipped: }, "denylist": ["openai/*"], // optional, none bundled; repo or machine "roles": { "scribe": "openrouter/typesafe/jev-1.13" }, // an agent in the roster; a keyed - "judgements": { "duplicate-match": "openrouter/typesafe/jev-1.13" } // provider's routes: machine only + "judgements": { "duplicate-match": "shadow:openrouter/typesafe/jev-1.13" } // provider's routes: machine only; + // host | shadow:/ | / } } ``` @@ -335,6 +336,57 @@ rather than skipped: one, applies in its place (ruling CD2 of 2026-09-29). The same fault in `~/.abcd.noindex/config.json` is refused, because that file is the person's own and a route they set is never dropped silently. +- **A judgement type's route has a mode** (itd-2609221009495079). An + `oracle.judgements.` value is `host`, `shadow:/` or + `/`. `host`, or no route, keeps the type on the host. A + repository's `host` holds against the machine's route to the same name, + since it only keeps work on the host. Only the person's own configuration + turns a type on: a bare `/` is honoured from + `~/.abcd.noindex/config.json`, the rule a role handed to a runner follows + (`runner.personal`). The same route + in a repository's `.abcd/config.json` is skipped with one diagnostic naming + the file and the route, and the machine's own route to the type applies in + its place, in its own mode, or the host when the machine sets none; so a + repository never turns a machine's shadow into on. A repository may still + set `host`, or `shadow:/`, which decides nothing. In + **shadow** the host still decides. + The decision adapter is asked the same typed question over the same state + beside it and decides nothing. Each pair (the host's answer, the adapter's + answer and probability, the state's SHA-256 and size) is appended to + `pairs.jsonl` in the newest lab home the repository's lane of + `~/.abcd.noindex/lab/` holds, so `abcd lab mint` comes first. A state larger + than the model's window (32K tokens, counted high at three bytes a token) is + never sent: its pair is recorded as skipped with the size. With no lab + minted, the host still decides and no pair is kept. A bare + `/` **turns the type on**: the adapter decides, and the run + record names the route and the model that answered. If the adapter cannot + answer (unreachable, failed, an answer outside the question's choices, a + state over its window), the host decides instead, with a receipt saying + why. A shadow route passes every rule a route turned on does (the + allowlist, the denylist, the keyed-provider skip, the malformed-route skip) + when the configuration is read and before any call. A role takes no mode. + The types are a closed registry: `remedy-read`, `duplicate-match`, + `unmeetable-criteria`, `eligibility-residual`, `source-classification`, + `source-selection`, `memory-check`, `ledger-category`, `ledger-severity`, + `prepass-overlap`, `present-tense-advisory`, `embark-lesson-ranking`. A + route naming anything else is + a diagnostic and is skipped. The drain's judgement before a lane opens runs + as `eligibility-residual`, a choice of `no`, `user-visible` or + `trust-rule`, and the drain's move calls the router for it + (`loop/drain_decide.go`). On the host, the default, the drain moves as it + always did. In shadow, the move records a pair once the host's or the + routed judge's answer is taken, and changes nothing else. Turned on, the + move asks the adapter once the host's request is written, before any runner + the drain's judge is routed to, and takes the answer the way a routed + judge's is taken, with the route named `decision-adapter /` + in the drain's record. If the adapter cannot answer, the host's request + stands, with a fallback receipt in the drain's state beside the routed + judge's, and the adapter is not asked again for that request. No other + type's step calls the router yet. Those judgements are made in the host's + command pages, which hand nothing back to the binary, so they stay on the + host whatever their route says until their step calls the router. `abcd lab harvest` reports the agreement per type, and + the research note filed from it is the evidence a ruling turns a type on + with. - **A route to a provider that holds a key sits on the machine alone.** Only a route the person set up on their own machine may spend their paid key (the product thinker's ruling AA(b) of 2026-09-29), so a repository's @@ -352,7 +404,9 @@ rather than skipped: provider holds a key when its block names `key`, judged from the block and never by reading the credential store. A repository's route to a provider whose block names no key (a local server) is admitted and wins over the - machine's per name, and a `--route` the person types is unaffected. + machine's per name, except a judgement type's route that turns the type on, + which only the machine's file may set (above), and a `--route` the person + types is unaffected. - **A call that spends a key takes no repository settings unless the person typed its route.** A provider leg sends the connection's defaults, then the winning routing row's settings, then the `--route`'s. On a leg to a provider that holds a key and diff --git a/.abcd/development/specs/open/spc-2609221011151661-abcd-s-typed-judgements-can-run-through.md b/.abcd/development/specs/open/spc-2609221011151661-abcd-s-typed-judgements-can-run-through.md index e2867c2679..bc0f9912de 100644 --- a/.abcd/development/specs/open/spc-2609221011151661-abcd-s-typed-judgements-can-run-through.md +++ b/.abcd/development/specs/open/spc-2609221011151661-abcd-s-typed-judgements-can-run-through.md @@ -30,7 +30,7 @@ One interface, one router with three modes, one adapter over the API adapter's c ## Footprint -- packages: internal/core/decide, internal/core/lab, the OpenAI-compatible client adapter, and callers across capture, intent, memory and lint +- packages: internal/core/decide, internal/core/lab, internal/adapter/openaiapi, internal/core/oracle/config.go, internal/core/oracle/judgement_mode_test.go, internal/core/oracle/connect_guide.go, internal/core/implement/loop/drain_decide.go, internal/core/implement/loop/drain_decide_test.go, internal/surface/cli/lab.go, internal/surface/cli/lab_surface_test.go, internal/README.md, commands/lab.md, .abcd/development/brief/04-surfaces/31-lab.md, .abcd/development/brief/05-internals/02-adapters.md, .abcd/development/brief/05-internals/03-configuration.md, .abcd/development/intents/planned/itd-2609221009495079-abcd-s-typed-judgements-can-run-through.md, .abcd/work/issues/open/iss-2610262033055327-the-decision-adapter-intent-treats.md, .abcd/work/issues/resolved/iss-2610262033055327-the-decision-adapter-intent-treats.md, .abcd/work/issues/open/iss-2610262037348735-the-decision-adapter-intent-needs-lab.md, .abcd/work/issues/resolved/iss-2610262037348735-the-decision-adapter-intent-needs-lab.md - tests: the router's three modes with a fake adapter; the pairs file; the harvest's rates; the window skip; the fallback receipt ## How the criteria are satisfied diff --git a/.abcd/work/issues/open/iss-2610262037348735-the-decision-adapter-intent-needs-lab.md b/.abcd/work/issues/resolved/iss-2610262037348735-the-decision-adapter-intent-needs-lab.md similarity index 79% rename from .abcd/work/issues/open/iss-2610262037348735-the-decision-adapter-intent-needs-lab.md rename to .abcd/work/issues/resolved/iss-2610262037348735-the-decision-adapter-intent-needs-lab.md index 69b8a41a2e..a11d0a26bd 100644 --- a/.abcd/work/issues/open/iss-2610262037348735-the-decision-adapter-intent-needs-lab.md +++ b/.abcd/work/issues/resolved/iss-2610262037348735-the-decision-adapter-intent-needs-lab.md @@ -12,6 +12,11 @@ found_at: ".abcd/development/intents/planned/itd-2609221009495079-abcd-s-typed-j remedy: "In the decision-adapter intent state that lab harvest gains a per-type agreement section computed from the adapter's recorded pairs (new scope on the shipped lab verb), or name a new sub-verb for it; the lab chapter describes a harvest that only assembles prose sections." related_intents: [itd-2609221009495079] finding_ref: cfk-98716f3a82450774 +resolution: "lab harvest gains a per-type agreement section computed from the decision adapter's pairs.jsonl (counts, compared, agreed, skipped, rate, four probability bands, and the research note's line naming the lab home and the count); the lab chapter now describes it" +impact: internal +resolved_by: + intent: "itd-2609221009495079" + spec: "spc-2609221011151661" --- The decision-adapter intent needs lab harvest to report agreement rates with probability bands; the lab brief's harvest only assembles prose sections from findings and probe records @@ -24,3 +29,7 @@ A sequencing impossibility (severity minor) found by the cross-document consiste The intent's evidence for turning a judgement type on is the agreement rates per type, with counts and probability bands, that `abcd lab harvest` produces from pairs the adapter recorded. In the lab chapter, the harvest assembles the lifeboat's sections from the INTENTION, the findings log and the probe records, and computes no statistics. The lab verbs run nothing either, because whoever runs the lab runs each probe. As the lab is written, its harvest cannot produce the figures this intent depends on. Evidence: `.abcd/work/reviews/2026-10-26-consistency/00-summary.md`, finding 733. + +## Grounds + +- pursued: the harvest's agreement section is the evidence path the intent names; shown wrong if a lab holding pairs harvests without per-type counts and bands diff --git a/commands/ahoy.md b/commands/ahoy.md index 526358ba06..59a068a340 100644 --- a/commands/ahoy.md +++ b/commands/ahoy.md @@ -1372,8 +1372,9 @@ adapter refuses as `oracle_api.config_refused`, naming the file and the key. A route the configuration read skips (a repository's route to a provider that holds a key, a repository's route that is not `/` or whose name is not a plain lower-case name, a repository's route to a model a provider -holding no key does not list, a route to a provider this machine has not -configured, or a role outside the roster) is the optional gap `oracle_api.route_skipped`, its +holding no key does not list, a repository's judgement route that turns a type +on, which only `~/.abcd.noindex/config.json` may set, a route to a provider this +machine has not configured, or a role outside the roster) is the optional gap `oracle_api.route_skipped`, its `detail` one line per skipped route; relay each line. A skipped repository route never displaces the machine's: where `~/.abcd.noindex/config.json` routes the same name, that route applies. diff --git a/commands/lab.md b/commands/lab.md index 7812738903..5485d24479 100644 --- a/commands/lab.md +++ b/commands/lab.md @@ -96,6 +96,23 @@ yourself**: show the candidates and let the user decide, and note any without a `refutation`. When a finding cites no probe record or evidence, or one that is missing or incomplete, `gaps` names it, the exit is 1 and nothing is written. +When the lab collected the decision adapter's shadow pairs, the harvest also +carries `agreement`. A judgement type is put in shadow by setting +`oracle.judgements.` to `shadow:/` in +`~/.abcd.noindex/config.json`. The host keeps deciding, and each answer the +adapter gives beside it is appended to `pairs.jsonl` in the newest lab home. +Today the drain's judgement before a lane opens (`eligibility-residual`) is +the one step that goes through the router, so it is the one type whose pairs +are kept; every other type stays on the host whatever its route says, and a +lab with no pairs has no `agreement`. +For each type, `agreement` gives `pairs`, `compared` (the adapter answered), +`agreed`, `skipped` (the state was over the model's window, or the call +failed), `rate`, and the compared and agreed counts in four probability +`bands`. It ends with the `note` line, naming the lab home and the count. Show +the user the rates and the note: a research note filed from it is the evidence +a ruling turns a type on with (`/`, no `shadow:`), and turning +a type on is the user's decision, never yours. + **Plan a model matrix run** in a lab whose preflight passed: ```bash diff --git a/internal/README.md b/internal/README.md index 15a39168b7..7bc54c69d1 100644 --- a/internal/README.md +++ b/internal/README.md @@ -237,6 +237,20 @@ plugin surface, and a future MCP server share one engine. already resolves. It owns the id, the date, the filename and the four sections, and states nothing: the decision is a human's to write, so the record lands `proposed` and the skeleton carries questions rather than answers. +- **`core/decide/judge/`** — the decision adapter's one interface + (itd-2609221009495079), apart from the ADR verb in spite of the shared name: a + typed judgement, (state, question) → (value, probability), whose question is + one of three kinds (choose, score, yes/no) and whose type is one of a closed + registry. The host's judgement (`Host`, probability 1) and the decision model + reached through `core/oracle`'s API call (`Adapter`) both implement it. A + `Router` reads `oracle.judgements.` as `host`, `shadow:/` + or `/`, which the oracle reader has already held to the + provider's allowlist. In shadow the host decides and the pair is recorded in + the newest lab home's `pairs.jsonl` (`core/lab`). Turned on, the adapter + decides, and an adapter that cannot answer leaves the decision to the host + with a receipt. The drain's judgement before a lane opens + (`eligibility-residual`, `core/implement/loop`) is the one step that calls + it; every other type stays on the host. - **`core/decisions/`** — the decision records' folder, `.abcd/work/decisions/`: one `dcn--.md` file per decision (adr-2610110925465593). `Record` is its one writer, behind `abcd decide ""`: it mints through diff --git a/internal/core/decide/judge/adapter.go b/internal/core/decide/judge/adapter.go new file mode 100644 index 0000000000..54dff2d29e --- /dev/null +++ b/internal/core/decide/judge/adapter.go @@ -0,0 +1,101 @@ +package judge + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "math" + "strings" + + "github.com/intentdriven/abcd/internal/adapter/openaiapi" + "github.com/intentdriven/abcd/internal/core/credential" + "github.com/intentdriven/abcd/internal/core/oracle" + "github.com/intentdriven/abcd/internal/termsafe" +) + +// Caller is one call through the OpenAI-compatible API adapter; +// *oracle.APIConfig implements it. Its Call admits the target against the +// allowlist and oracle.denylist again and resolves the key by name, so a +// target built anywhere is held to adr-2609221009491186 before any call. +type Caller interface { + Call(ctx context.Context, creds credential.Source, req oracle.CallRequest, opts ...openaiapi.Option) ([]byte, oracle.CallRecord, error) +} + +// Adapter is the decision model through the one interface: the question is +// rendered as the host's is, sent to Target through Calls, and the answer is +// read as a value and a probability. The model cannot abstain, so an answer +// outside the question's choices, or a probability outside 0 to 1, is no +// answer at all. +type Adapter struct { + Calls Caller + Creds credential.Source + Target oracle.Target + Options []openaiapi.Option +} + +// adapterInstructions frame every typed question the adapter is sent. +const adapterInstructions = "You answer one typed judgement. Read the question and the state, then reply with " + + "a single JSON object and nothing else: {\"value\": \"\", " + + "\"probability\": }." + +// maxAnswerBytes bounds the answer the adapter parses: a value and a number. +const maxAnswerBytes = 4 << 10 + +// typed is the answer's shape. +type typed struct { + Value *string `json:"value"` + Probability *float64 `json:"probability"` +} + +// Judge sends the question through the API adapter and parses the answer. +func (a Adapter) Judge(ctx context.Context, s State, q Question) (Answer, error) { + if err := q.Check(); err != nil { + return Answer{}, err + } + if a.Calls == nil { + return Answer{}, fmt.Errorf("judge: no adapter configured for %s", q.Type) + } + var got Answer + contract := func(content []byte) error { + v, err := parseTyped(q, content) + got = v + return err + } + brief := openaiapi.Brief{Instructions: adapterInstructions, Input: render(s, q)} + _, rec, err := a.Calls.Call(ctx, a.Creds, oracle.CallRequest{Target: a.Target, Brief: brief, Contract: contract}, a.Options...) + if err != nil { + return Answer{}, err + } + got.Model = rec.ModelReported + return got, nil +} + +// parseTyped reads the adapter's answer: one JSON object, perhaps inside a +// code fence, whose value is one of the question's choices and whose +// probability is a number from 0 to 1. +func parseTyped(q Question, content []byte) (Answer, error) { + if len(content) > maxAnswerBytes { + return Answer{}, fmt.Errorf("judge: the adapter's answer is %d bytes, more than the %d a typed answer takes", len(content), maxAnswerBytes) + } + body := bytes.TrimSpace(content) + if i, j := bytes.IndexByte(body, '{'), bytes.LastIndexByte(body, '}'); i >= 0 && j > i { + body = body[i : j+1] + } + var t typed + if err := json.Unmarshal(body, &t); err != nil { + return Answer{}, fmt.Errorf("judge: the adapter's answer is not a typed answer: %s", termsafe.Sanitize(err.Error())) + } + if t.Value == nil || t.Probability == nil { + return Answer{}, fmt.Errorf("judge: the adapter's answer lacks its value or its probability") + } + p := *t.Probability + if math.IsNaN(p) || p < 0 || p > 1 { + return Answer{}, fmt.Errorf("judge: the adapter's probability %v is not from 0 to 1", p) + } + v, ok := q.match(*t.Value) + if !ok { + return Answer{}, fmt.Errorf("judge: the adapter's answer %q is not one of %s", termsafe.Sanitize(clip(*t.Value)), strings.Join(q.Choices(), ", ")) + } + return Answer{Value: v, Probability: p}, nil +} diff --git a/internal/core/decide/judge/host.go b/internal/core/decide/judge/host.go new file mode 100644 index 0000000000..eb0436a7fb --- /dev/null +++ b/internal/core/decide/judge/host.go @@ -0,0 +1,52 @@ +package judge + +import ( + "context" + "errors" + "fmt" + + "github.com/intentdriven/abcd/internal/termsafe" +) + +// ErrHostPending is a host judgement asked out of band: the caller has +// written the host's request (the drain's request file) and the answer comes +// back in a later move. A Router passes it through, with its receipt. +var ErrHostPending = errors.New("judge: the host's answer is awaited") + +// Host is the host's own judgement through the one interface (adr-25): Ask is +// shown the rendered prompt, the same text the adapter is sent, and returns +// the host's answer, which is parsed against the question's choices at +// probability 1. A caller whose host has already answered uses Answered. +type Host struct { + Ask func(ctx context.Context, prompt string) (string, error) +} + +// Answered is a Host whose answer is already in hand: the drain's host has +// written its answer file, or a page's host has handed its judgement back. +func Answered(text string) Host { + return Host{Ask: func(context.Context, string) (string, error) { return text, nil }} +} + +// Pending is a Host asked out of band: its Judge returns ErrHostPending. +func Pending() Host { + return Host{Ask: func(context.Context, string) (string, error) { return "", ErrHostPending }} +} + +// Judge asks the host and parses its answer at probability 1. +func (h Host) Judge(ctx context.Context, s State, q Question) (Answer, error) { + if err := q.Check(); err != nil { + return Answer{}, err + } + if h.Ask == nil { + return Answer{}, ErrHostPending + } + text, err := h.Ask(ctx, render(s, q)) + if err != nil { + return Answer{}, err + } + v, ok := q.match(text) + if !ok { + return Answer{}, fmt.Errorf("judge: the host's answer %q to %s is not one of %v", termsafe.Sanitize(clip(text)), q.Type, q.Choices()) + } + return Answer{Value: v, Probability: 1}, nil +} diff --git a/internal/core/decide/judge/judge.go b/internal/core/decide/judge/judge.go new file mode 100644 index 0000000000..50df3e16fa --- /dev/null +++ b/internal/core/decide/judge/judge.go @@ -0,0 +1,220 @@ +// Package judge is the decision adapter's one interface (itd-2609221009495079, +// spc-2609221011151661 scope 1): a typed judgement, (state, question) → +// (value, probability), with three question kinds — choose one of N, score on +// a scale, yes or no. The host's own judgement implements it (Host, at +// probability 1), and so does the decision model reached through the +// OpenAI-compatible API adapter (Adapter), so the two are compared like for +// like and a judgement type is switched between them by configuration alone +// (router.go), never by its caller. +// +// It sits apart from internal/core/decide, the ADR verb's package: the name is +// shared, the concern is not. +// +// Like the rest of the core it never prints. It writes only through the +// recorder its Router is given, which is the lab's pairs file in the +// machine-scoped store (internal/core/lab), never the repository. +package judge + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "slices" + "strconv" + "strings" +) + +// Kind is a question's kind. +type Kind string + +// The three question kinds. +const ( + // KindChoose asks for one of the question's options. + KindChoose Kind = "choose" + // KindScore asks for an integer on the question's scale, Min to Max. + KindScore Kind = "score" + // KindYesNo asks yes or no. + KindYesNo Kind = "yes-no" +) + +// Type is a judgement type: one judgement-shaped step abcd may route through +// the adapter. The registry is closed: a type outside it is refused. +type Type string + +// The judgement types the intent's scope lists. Embark's lesson ranking is +// one: it is to run through the adapter in place of the term-overlap ordering +// the embark verb uses today (iss-2610262033055327), and like every other +// type it is decided by the host until its step calls the router. +const ( + RemedyRead Type = "remedy-read" + DuplicateMatch Type = "duplicate-match" + UnmeetableCriteria Type = "unmeetable-criteria" + EligibilityResidual Type = "eligibility-residual" + SourceClassification Type = "source-classification" + SourceSelection Type = "source-selection" + MemoryCheck Type = "memory-check" + LedgerCategory Type = "ledger-category" + LedgerSeverity Type = "ledger-severity" + PrepassOverlap Type = "prepass-overlap" + PresentTense Type = "present-tense-advisory" + EmbarkLessonRanking Type = "embark-lesson-ranking" +) + +var registry = []Type{RemedyRead, DuplicateMatch, UnmeetableCriteria, EligibilityResidual, SourceClassification, + SourceSelection, MemoryCheck, LedgerCategory, LedgerSeverity, PrepassOverlap, PresentTense, EmbarkLessonRanking} + +// types returns the registry, in its order. +func types() []Type { return slices.Clone(registry) } + +// known reports whether name is a type in the registry. +func known(name string) bool { return slices.Contains(registry, Type(name)) } + +// Bounds on a question. +const ( + maxOptions = 64 + maxOption = 200 + maxAsk = 2000 + maxScaleLen = 100 +) + +// Question is one typed question. +type Question struct { + Type Type + Kind Kind + // Ask is the question in words. + Ask string + // Options are a choose question's options. + Options []string + // Min and Max are a score question's scale, inclusive. + Min, Max int +} + +// Choices are the answers the question admits, as an answer spells them. +func (q Question) Choices() []string { + switch q.Kind { + case KindYesNo: + return []string{"yes", "no"} + case KindScore: + out := make([]string, 0, q.Max-q.Min+1) + for v := q.Min; v <= q.Max; v++ { + out = append(out, strconv.Itoa(v)) + } + return out + default: + return slices.Clone(q.Options) + } +} + +// Check refuses a question that does not hold together: a type outside the +// registry, an unknown kind, no words, or options or a scale out of shape. +func (q Question) Check() error { + if !known(string(q.Type)) { + return fmt.Errorf("judge: %q is not a judgement type (the registry is %s)", clip(string(q.Type)), typeNames()) + } + if strings.TrimSpace(q.Ask) == "" || len(q.Ask) > maxAsk { + return fmt.Errorf("judge: a %s question needs its words (at most %d bytes)", q.Type, maxAsk) + } + switch q.Kind { + case KindYesNo: + case KindScore: + if q.Max <= q.Min || q.Max-q.Min >= maxScaleLen { + return fmt.Errorf("judge: a %s score question needs a scale of 2 to %d points, not %d to %d", q.Type, maxScaleLen, q.Min, q.Max) + } + case KindChoose: + if len(q.Options) < 1 || len(q.Options) > maxOptions { + return fmt.Errorf("judge: a %s choose question needs 1 to %d options, not %d", q.Type, maxOptions, len(q.Options)) + } + seen := map[string]bool{} + for _, o := range q.Options { + k := strings.ToLower(strings.TrimSpace(o)) + if k == "" || len(o) > maxOption || strings.ContainsAny(o, "\n\r") || seen[k] { + return fmt.Errorf("judge: a %s choose question's options are distinct one-line words of at most %d bytes", q.Type, maxOption) + } + seen[k] = true + } + default: + return fmt.Errorf("judge: %q is not a question kind (choose, score, yes-no)", clip(string(q.Kind))) + } + return nil +} + +// match returns the choice text names, matched without regard to case or +// surrounding space, and whether it is one. +func (q Question) match(text string) (string, bool) { + t := strings.TrimSpace(text) + for _, c := range q.Choices() { + if strings.EqualFold(c, t) { + return c, true + } + } + return "", false +} + +// State is what a judgement is made over, after retrieval: the text both +// implementations are shown. +type State struct { + Text string +} + +// SHA256 is the state's hash, the identity a pair records instead of the text. +func (s State) SHA256() string { + sum := sha256.Sum256([]byte(s.Text)) + return hex.EncodeToString(sum[:]) +} + +// Answer is a judgement's answer: one of the question's choices, the +// probability the answering side gives it, and, from the adapter, the model +// the provider reported. +type Answer struct { + Value string + Probability float64 + Model string +} + +// Judge is the one interface: the host and the adapter both implement it. +type Judge interface { + Judge(ctx context.Context, s State, q Question) (Answer, error) +} + +// render is the prompt a judgement is asked as, the same for the host and the +// adapter. +func render(s State, q Question) string { + var b strings.Builder + fmt.Fprintf(&b, "Judgement: %s (%s)\n\nQuestion: %s\n\n", q.Type, q.Kind, strings.TrimSpace(q.Ask)) + switch q.Kind { + case KindChoose: + b.WriteString("Answer with exactly one of these options:\n") + for _, o := range q.Options { + fmt.Fprintf(&b, "- %s\n", o) + } + case KindScore: + fmt.Fprintf(&b, "Answer with one whole number from %d to %d.\n", q.Min, q.Max) + case KindYesNo: + b.WriteString("Answer yes or no.\n") + } + fmt.Fprintf(&b, "\nState:\n\n%s\n", s.Text) + return b.String() +} + +// tokens estimates the tokens text takes, high rather than low: three bytes a +// token, where English prose runs nearer four, so a state that fits by this +// count fits the model's window. +func tokens(text string) int { return (len(text) + 2) / 3 } + +func typeNames() string { + out := make([]string, len(registry)) + for i, t := range registry { + out[i] = string(t) + } + return strings.Join(out, ", ") +} + +// clip bounds a caller-supplied string for a message. +func clip(s string) string { + const n = 64 + if len(s) > n { + return s[:n] + "…" + } + return s +} diff --git a/internal/core/decide/judge/judge_test.go b/internal/core/decide/judge/judge_test.go new file mode 100644 index 0000000000..d2fd7a3478 --- /dev/null +++ b/internal/core/decide/judge/judge_test.go @@ -0,0 +1,394 @@ +package judge + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/intentdriven/abcd/internal/abcdhome" + "github.com/intentdriven/abcd/internal/adapter/openaiapi" + "github.com/intentdriven/abcd/internal/core/credential" + "github.com/intentdriven/abcd/internal/core/lab" + "github.com/intentdriven/abcd/internal/core/layered" + "github.com/intentdriven/abcd/internal/core/oracle" + "github.com/intentdriven/abcd/internal/gittest" +) + +// The decision adapter (itd-2609221009495079, spc-2609221011151661): one +// interface the host and the adapter both implement, and one router with three +// modes per judgement type. The adapter here is a fake answering through the +// same interface; TestTheAPIAdapterParsesTheTypedAnswer drives the real +// Adapter over a fake oracle caller. + +var target = oracle.Target{Provider: "openrouter", Model: "typesafe/jev-1.13", Origin: abcdhome.Display("config.json")} + +func dupQuestion() Question { + return Question{Type: DuplicateMatch, Kind: KindChoose, Ask: "Which record does this capture duplicate?", + Options: []string{"none", "iss-1", "iss-2"}} +} + +// fake is a Judge that answers value at p, or fails with err, and counts calls. +type fake struct { + value string + p float64 + err error + calls int + seen []string +} + +func (f *fake) Judge(_ context.Context, s State, q Question) (Answer, error) { + f.calls++ + f.seen = append(f.seen, render(s, q)) + if f.err != nil { + return Answer{}, f.err + } + return Answer{Value: f.value, Probability: f.p, Model: "typesafe/jev-1.13-0901"}, nil +} + +// pairs is a recorder that keeps what it is given. +type pairs struct { + got []lab.Pair + err error +} + +func (p *pairs) record(pr lab.Pair) (string, string, error) { + if p.err != nil { + return "", "", p.err + } + p.got = append(p.got, pr) + return "lab-261029000000-abcdef0", abcdhome.Display("lab/x/lab-261029000000-abcdef0/pairs.jsonl"), nil +} + +func router(mode string, adapter *fake, rec *pairs) *Router { + routes := map[Type]oracle.Target{} + if mode != oracle.JudgementHost { + t := target + t.Mode = mode + routes[DuplicateMatch] = t + } + return newRouter(routes, func(oracle.Target) Judge { return adapter }, rec.record) +} + +// Criterion 1: shadow — the host decides, the adapter answers beside it, and +// the pair with its probability and the state's hash is recorded. +func TestShadowTheHostDecidesAndThePairIsRecorded(t *testing.T) { + adapter, rec := &fake{value: "iss-2", p: 0.81}, &pairs{} + host := Answered("iss-1") + s := State{Text: "capture: the parser drops a trailing comma"} + a, rc, err := router(oracle.JudgementShadow, adapter, rec).Decide(context.Background(), s, dupQuestion(), host) + if err != nil { + t.Fatal(err) + } + if a.Value != "iss-1" || a.Probability != 1 || rc.Route != RouteHost || rc.Mode != oracle.JudgementShadow { + t.Fatalf("answer %+v receipt %+v: the host decides in shadow", a, rc) + } + if adapter.calls != 1 || len(rec.got) != 1 { + t.Fatalf("adapter calls %d, pairs %d; want one each", adapter.calls, len(rec.got)) + } + p := rec.got[0] + if p.Type != string(DuplicateMatch) || p.Kind != string(KindChoose) || p.Host != "iss-1" || p.Adapter != "iss-2" || + p.Probability == nil || *p.Probability != 0.81 || p.StateSHA256 != s.SHA256() || p.Target != "openrouter/typesafe/jev-1.13" || + p.ModelReported != "typesafe/jev-1.13-0901" || p.Skipped != "" { + t.Fatalf("pair = %+v", p) + } + if !strings.Contains(rc.Pair, "pairs.jsonl") { + t.Fatalf("receipt pair = %q", rc.Pair) + } +} + +// Criterion 1, end to end: the pair lands in pairs.jsonl under the newest lab +// home in the machine-scoped store, and nothing is written into the +// repository. +func TestShadowWritesThePairUnderTheLabHomeAndNothingInTheRepository(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + r := gittest.NewRepo(t) + r.Write("README.md", "root\n") + r.Commit("root") + t.Cleanup(lab.SetAbcdRootCommitForTest(strings.Repeat("a", 40))) + m, err := lab.Mint(r.Root(), "does the decision adapter agree with the host?", "") + if err != nil { + t.Fatal(err) + } + before := r.Git("status", "--porcelain", "--ignored") + t2 := target + t2.Mode = oracle.JudgementShadow + rt := newRouter(map[Type]oracle.Target{DuplicateMatch: t2}, func(oracle.Target) Judge { return &fake{value: "none", p: 0.7} }, + func(p lab.Pair) (string, string, error) { return lab.RecordPair(r.Root(), p) }) + if _, _, err := rt.Decide(context.Background(), State{Text: "s"}, dupQuestion(), Answered("none")); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(abcdhome.Path(home, "lab", m.RootSHA, m.ID, "pairs.jsonl")) + if err != nil { + t.Fatalf("pairs.jsonl: %v", err) + } + var p lab.Pair + if err := json.Unmarshal(data, &p); err != nil || p.Host != "none" || p.Adapter != "none" { + t.Fatalf("pair = %s (%v)", data, err) + } + if after := r.Git("status", "--porcelain", "--ignored"); after != before { + t.Fatalf("the repository changed:\n%s", after) + } +} + +// Criterion 5: a state larger than the window is recorded as skipped with its +// size, the adapter is never called, and the host decides alone. +func TestShadowSkipsAStateLargerThanTheWindow(t *testing.T) { + adapter, rec := &fake{value: "iss-2", p: 0.9}, &pairs{} + rt := router(oracle.JudgementShadow, adapter, rec) + rt.Window = 100 + s := State{Text: strings.Repeat("word ", 400)} + a, rc, err := rt.Decide(context.Background(), s, dupQuestion(), Answered("iss-1")) + if err != nil || a.Value != "iss-1" || rc.Route != RouteHost { + t.Fatalf("answer %+v, receipt %+v, err %v", a, rc, err) + } + if adapter.calls != 0 { + t.Fatal("the adapter was called on a state larger than its window") + } + if len(rec.got) != 1 || rec.got[0].Skipped != lab.SkippedWindow || rec.got[0].StateTokens <= 100 || + !strings.Contains(rec.got[0].Detail, "window of 100") || rec.got[0].Adapter != "" { + t.Fatalf("pairs = %+v", rec.got) + } + if rc.PairSkipped != lab.SkippedWindow || rc.StateTokens != rec.got[0].StateTokens { + t.Fatalf("receipt = %+v", rc) + } +} + +// A shadow adapter that fails changes nothing for the host's decision; the +// pair is recorded as skipped, naming why. +func TestShadowRecordsAFailedAdapterAndTheHostStillDecides(t *testing.T) { + adapter, rec := &fake{err: errors.New("connection refused")}, &pairs{} + a, _, err := router(oracle.JudgementShadow, adapter, rec).Decide(context.Background(), State{Text: "s"}, dupQuestion(), Answered("iss-2")) + if err != nil || a.Value != "iss-2" { + t.Fatalf("answer %+v, err %v", a, err) + } + if len(rec.got) != 1 || rec.got[0].Skipped != lab.SkippedFailed || !strings.Contains(rec.got[0].Detail, "connection refused") { + t.Fatalf("pairs = %+v", rec.got) + } +} + +// No lab minted: the host decides, and the receipt says the pair was not kept. +func TestShadowWithNoLabStillDecidesAndSaysSo(t *testing.T) { + rec := &pairs{err: lab.ErrNoLab} + a, rc, err := router(oracle.JudgementShadow, &fake{value: "none", p: 0.5}, rec).Decide(context.Background(), State{Text: "s"}, dupQuestion(), Answered("none")) + if err != nil || a.Value != "none" || !strings.Contains(rc.Note, "no lab minted") { + t.Fatalf("answer %+v, receipt %+v, err %v", a, rc, err) + } +} + +// Criterion 3: turned on, the adapter decides and the receipt names the route. +func TestOnTheAdapterDecidesAndTheReceiptNamesTheRoute(t *testing.T) { + adapter, rec := &fake{value: "iss-2", p: 0.93}, &pairs{} + host := &fake{value: "iss-1", p: 1} + a, rc, err := router(oracle.JudgementOn, adapter, rec).Decide(context.Background(), State{Text: "s"}, dupQuestion(), host) + if err != nil { + t.Fatal(err) + } + if a.Value != "iss-2" || a.Probability != 0.93 || host.calls != 0 { + t.Fatalf("answer %+v, host calls %d: the adapter decides", a, host.calls) + } + if rc.Route != RouteAdapter || rc.Target != "openrouter/typesafe/jev-1.13" || rc.ModelReported != "typesafe/jev-1.13-0901" || + rc.Fallback != "" || len(rec.got) != 0 { + t.Fatalf("receipt %+v, pairs %d", rc, len(rec.got)) + } +} + +// Criterion 3: an unreachable adapter falls back to the host with a receipt +// saying so. +func TestOnAnUnreachableAdapterFallsBackToTheHostWithAReceipt(t *testing.T) { + unreachable := &fake{err: openaiapi.ErrUnreachable} + host := &fake{value: "iss-1", p: 1} + a, rc, err := router(oracle.JudgementOn, unreachable, &pairs{}).Decide(context.Background(), State{Text: "s"}, dupQuestion(), host) + if err != nil || a.Value != "iss-1" || host.calls != 1 { + t.Fatalf("answer %+v, host calls %d, err %v", a, host.calls, err) + } + if rc.Route != RouteHost || !strings.Contains(rc.Fallback, "openrouter/typesafe/jev-1.13") || !strings.Contains(rc.Fallback, "the host decided") { + t.Fatalf("receipt = %+v", rc) + } + // A host asked out of band (the drain's request file) is pending: the + // fallback is still in the receipt. + _, rc, err = router(oracle.JudgementOn, unreachable, &pairs{}).Decide(context.Background(), State{Text: "s"}, dupQuestion(), Pending()) + if !errors.Is(err, ErrHostPending) || rc.Fallback == "" { + t.Fatalf("pending host: receipt %+v, err %v", rc, err) + } + // A state over the window is a fallback too, never a call. + rt := router(oracle.JudgementOn, unreachable, &pairs{}) + rt.Window = 10 + unreachable.calls = 0 + _, rc, _ = rt.Decide(context.Background(), State{Text: strings.Repeat("x", 400)}, dupQuestion(), host) + if unreachable.calls != 0 || !strings.Contains(rc.Fallback, "window") { + t.Fatalf("over-window on: calls %d, receipt %+v", unreachable.calls, rc) + } +} + +// Criterion 6: one interface, so a type is switched by configuration alone. +// The same caller, the same arguments: host, shadow, on. +func TestOneInterfaceSwitchesATypeWithoutTheCallerChanging(t *testing.T) { + caller := func(rt *Router) (Answer, Receipt) { + a, rc, err := rt.Decide(context.Background(), State{Text: "s"}, dupQuestion(), Answered("iss-1")) + if err != nil { + t.Fatal(err) + } + return a, rc + } + for _, c := range []struct { + mode, value, route string + pairs int + }{ + {oracle.JudgementHost, "iss-1", RouteHost, 0}, + {oracle.JudgementShadow, "iss-1", RouteHost, 1}, + {oracle.JudgementOn, "iss-2", RouteAdapter, 0}, + } { + adapter, rec := &fake{value: "iss-2", p: 0.9}, &pairs{} + a, rc := caller(router(c.mode, adapter, rec)) + if a.Value != c.value || rc.Route != c.route || len(rec.got) != c.pairs || rc.Mode != c.mode { + t.Errorf("%s: answer %+v receipt %+v pairs %d", c.mode, a, rc, len(rec.got)) + } + if c.mode == oracle.JudgementHost && adapter.calls != 0 { + t.Errorf("host: the adapter was called") + } + } + // Both implementations satisfy the one interface. + var _ Judge = Host{} + var _ Judge = Adapter{} +} + +// The registry is closed: a type outside it, or a question that does not hold +// together, is refused before anything is asked. +func TestTheRegistryIsClosedAndAQuestionIsChecked(t *testing.T) { + rt := router(oracle.JudgementOn, &fake{value: "x"}, &pairs{}) + host := &fake{value: "none"} + if _, _, err := rt.Decide(context.Background(), State{Text: "s"}, Question{Type: "lesson-polish", Kind: KindChoose, Ask: "?", Options: []string{"a"}}, host); err == nil { + t.Fatal("a type outside the registry was asked") + } + if _, _, err := rt.Decide(context.Background(), State{Text: "s"}, Question{Type: DuplicateMatch, Kind: KindChoose, Ask: "?"}, host); err == nil { + t.Fatal("a choose question with no options was asked") + } + if host.calls != 0 { + t.Fatal("the host was asked a refused question") + } + for _, ty := range types() { + if !known(string(ty)) { + t.Errorf("%s is listed but not known", ty) + } + } + // Embark's lesson ranking is a judgement routed through the adapter in + // place of its term-overlap ordering (iss-2610262033055327). + if !known(string(EmbarkLessonRanking)) || EmbarkLessonRanking != "embark-lesson-ranking" { + t.Error("embark's lesson ranking is not in the registry") + } +} + +// An adapter's answer outside the question's options is not an answer: turned +// on, the host decides instead, with a receipt. +func TestOnAnAnswerOutsideTheOptionsFallsBack(t *testing.T) { + host := &fake{value: "none"} + a, rc, err := router(oracle.JudgementOn, &fake{value: "iss-9", p: 0.99}, &pairs{}).Decide(context.Background(), State{Text: "s"}, dupQuestion(), host) + if err != nil || a.Value != "none" || !strings.Contains(rc.Fallback, "not one of") { + t.Fatalf("answer %+v receipt %+v err %v", a, rc, err) + } +} + +func TestTheHostParsesItsAnswerAtProbabilityOne(t *testing.T) { + for _, c := range []struct { + q Question + text string + want string + bad bool + }{ + {dupQuestion(), " ISS-2 \n", "iss-2", false}, + {dupQuestion(), "iss-7", "", true}, + {Question{Type: RemedyRead, Kind: KindYesNo, Ask: "Does it change what a user sees?"}, "Yes", "yes", false}, + {Question{Type: LedgerSeverity, Kind: KindScore, Ask: "How severe?", Min: 1, Max: 5}, "4", "4", false}, + {Question{Type: LedgerSeverity, Kind: KindScore, Ask: "How severe?", Min: 1, Max: 5}, "9", "", true}, + } { + a, err := Answered(c.text).Judge(context.Background(), State{Text: "s"}, c.q) + if c.bad { + if err == nil { + t.Errorf("%q: parsed as %+v, want a refusal", c.text, a) + } + continue + } + if err != nil || a.Value != c.want || a.Probability != 1 { + t.Errorf("%q: %+v, %v", c.text, a, err) + } + } + // The host is asked the prompt the adapter is asked: like for like. + var asked string + h := Host{Ask: func(_ context.Context, prompt string) (string, error) { asked = prompt; return "none", nil }} + if _, err := h.Judge(context.Background(), State{Text: "the state"}, dupQuestion()); err != nil { + t.Fatal(err) + } + if asked != render(State{Text: "the state"}, dupQuestion()) || !strings.Contains(asked, "the state") || !strings.Contains(asked, "iss-2") { + t.Fatalf("host prompt = %q", asked) + } +} + +// caller is a fake oracle call: it checks the request and answers content. +type caller struct { + content string + req oracle.CallRequest +} + +func (c *caller) Call(_ context.Context, _ credential.Source, req oracle.CallRequest, _ ...openaiapi.Option) ([]byte, oracle.CallRecord, error) { + c.req = req + if req.Contract != nil { + if err := req.Contract([]byte(c.content)); err != nil { + return nil, oracle.CallRecord{}, err + } + } + return []byte(c.content), oracle.CallRecord{Provider: req.Target.Provider, ModelAsked: req.Target.Model, ModelReported: "typesafe/jev-1.13-0901"}, nil +} + +func TestTheAPIAdapterParsesTheTypedAnswer(t *testing.T) { + c := &caller{content: "```json\n{\"value\": \"iss-2\", \"probability\": 0.87}\n```"} + a, err := Adapter{Calls: c, Target: target}.Judge(context.Background(), State{Text: "s"}, dupQuestion()) + if err != nil || a.Value != "iss-2" || a.Probability != 0.87 || a.Model != "typesafe/jev-1.13-0901" { + t.Fatalf("answer %+v, err %v", a, err) + } + if c.req.Target != target || !strings.Contains(c.req.Brief.Input, "iss-1") || !strings.Contains(c.req.Brief.Instructions, "probability") { + t.Fatalf("request = %+v", c.req) + } + for _, bad := range []string{`{"value":"iss-2","probability":1.5}`, `{"value":"iss-9","probability":0.5}`, `not json`, `{"value":"iss-2"}`} { + c.content = bad + if a, err := (Adapter{Calls: c, Target: target}).Judge(context.Background(), State{Text: "s"}, dupQuestion()); err == nil { + t.Errorf("%s: parsed as %+v", bad, a) + } + } +} + +// Criterion 4: a configured model not on its provider's list is refused when +// the configuration is read, before any call; a listed one loads with its mode. +func TestLoadRefusesAnUnlistedModelBeforeAnyCall(t *testing.T) { + roots := layered.Roots{Repo: t.TempDir(), Home: t.TempDir()} + write := func(body string) { + p := abcdhome.Path(roots.Home, "config.json") + if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + block := `"api":{"openrouter":{"base_url":"https://openrouter.ai/api/v1","key":"openrouter","models":["typesafe/jev-1.13"]}}` + write(`{"oracle":{` + block + `,"judgements":{"duplicate-match":"shadow:openrouter/mistral/small"}}}`) + if _, err := Load(roots, roots.Repo); err == nil || !strings.Contains(err.Error(), "refused before any call") { + t.Fatalf("Load = %v, want the allowlist refusal", err) + } + write(`{"oracle":{` + block + `,"judgements":{"duplicate-match":"shadow:openrouter/typesafe/jev-1.13", + "remedy-read":"openrouter/typesafe/jev-1.13","lesson-polish":"openrouter/typesafe/jev-1.13"}}}`) + rt, err := Load(roots, roots.Repo) + if err != nil { + t.Fatal(err) + } + if rt.Mode(DuplicateMatch) != oracle.JudgementShadow || rt.Mode(RemedyRead) != oracle.JudgementOn || rt.Mode(MemoryCheck) != oracle.JudgementHost { + t.Fatalf("modes: %s %s %s", rt.Mode(DuplicateMatch), rt.Mode(RemedyRead), rt.Mode(MemoryCheck)) + } + if len(rt.Diagnostics) != 1 || !strings.Contains(rt.Diagnostics[0], "lesson-polish") { + t.Fatalf("diagnostics = %q, want the unknown type named", rt.Diagnostics) + } +} diff --git a/internal/core/decide/judge/router.go b/internal/core/decide/judge/router.go new file mode 100644 index 0000000000..95d14f1905 --- /dev/null +++ b/internal/core/decide/judge/router.go @@ -0,0 +1,257 @@ +package judge + +// router.go is the per-type switch (spc-2609221011151661 scopes 3, 5 and 6). +// Each judgement type is routed as oracle.judgements. says, read and +// checked against the provider's allowlist when the configuration is loaded, +// before any call (adr-2609221009491186, criterion 4): +// +// - host, or nothing configured: the host decides, and nothing else runs. +// - shadow:/: the host decides; the adapter answers beside +// it and decides nothing; the pair is recorded in the newest lab home +// (criterion 1). A state larger than the model's window is not sent: the +// pair is recorded as skipped with its size (criterion 5). +// - /: the adapter decides and the receipt names the +// route; an adapter that cannot answer — unreachable, failed, an answer +// outside the question's choices, a state over its window — leaves the +// host to decide, with a receipt saying why (criterion 3). +// +// The caller hands Decide its host judgement and nothing else changes when a +// type is switched (criterion 6). + +import ( + "context" + "errors" + "fmt" + "math" + "strings" + + "github.com/intentdriven/abcd/internal/core/credential" + "github.com/intentdriven/abcd/internal/core/lab" + "github.com/intentdriven/abcd/internal/core/layered" + "github.com/intentdriven/abcd/internal/core/oracle" + "github.com/intentdriven/abcd/internal/termsafe" +) + +// DefaultWindow is the decision model's context, in tokens: the first +// candidate's 32K (itd-2609221009495079 scope condition 1). +const DefaultWindow = 32 << 10 + +// The routes a receipt names. +const ( + RouteHost = "host" + RouteAdapter = "adapter" +) + +// maxDetail bounds an error's account in a pair or a receipt. +const maxDetail = 300 + +// Receipt is what a judgement's run record carries: the type and its mode, the +// route that decided, the adapter's target and the model it reported, the +// probability the deciding side gave, the state's hash and size, why a type +// turned on fell back to the host, and where a shadow pair went or why it was +// not kept. +type Receipt struct { + Type Type `json:"type"` + Mode string `json:"mode"` + Route string `json:"route"` + Target string `json:"target,omitempty"` + ModelReported string `json:"model_reported,omitempty"` + Probability float64 `json:"probability"` + StateSHA256 string `json:"state_sha256"` + StateTokens int `json:"state_tokens"` + Fallback string `json:"fallback,omitempty"` + Pair string `json:"pair,omitempty"` + PairSkipped string `json:"pair_skipped,omitempty"` + Note string `json:"note,omitempty"` +} + +// Recorder keeps a shadow pair and returns the lab and the file it went to. +type Recorder func(lab.Pair) (labID, where string, err error) + +// Router routes each judgement type to the host, to the host with the adapter +// in shadow, or to the adapter. +type Router struct { + routes map[Type]oracle.Target + adapter func(oracle.Target) Judge + record Recorder + // Window is the adapter's window in tokens; DefaultWindow when zero. + Window int + // Diagnostics are the configuration's non-fatal reports, for a front + // door to print: the oracle reader's, and a route naming a type outside + // the registry. + Diagnostics []string +} + +// newRouter is a Router over routes (a type absent from it stays on the +// host), the adapter each target is judged by, and where shadow pairs are +// recorded. +func newRouter(routes map[Type]oracle.Target, adapter func(oracle.Target) Judge, record Recorder) *Router { + r := &Router{routes: map[Type]oracle.Target{}, adapter: adapter, record: record} + for t, tgt := range routes { + r.routes[t] = tgt + } + return r +} + +// NewForTest is newRouter, the seam a caller's tests route a type with +// through a fake adapter and recorder. +func NewForTest(routes map[Type]oracle.Target, adapter func(oracle.Target) Judge, record Recorder) *Router { + return newRouter(routes, adapter, record) +} + +// Load reads oracle.judgements. through the oracle reader, which refuses +// a model not on its provider's allowlist (or one oracle.denylist matches) +// before any call, and returns the Router the repository at repoRoot judges +// with: the adapter is the API adapter with the key resolved by name from the +// account's credential store, and shadow pairs go to the newest lab minted for +// the repository. A route to a type outside the registry is a diagnostic. +func Load(roots layered.Roots, repoRoot string) (*Router, error) { + cfg, err := oracle.LoadAPI(roots) + if err != nil { + return nil, err + } + routes := map[Type]oracle.Target{} + var diags []string + diags = append(diags, cfg.Diagnostics...) + for _, pr := range cfg.Routes() { + if pr.Kind != "judgement" { + continue + } + if !known(pr.Name) { + diags = append(diags, fmt.Sprintf("decision adapter: oracle.judgements.%s in %s is not a judgement type (%s); it is skipped", + termsafe.Sanitize(clip(pr.Name)), termsafe.Sanitize(pr.Target.Origin), typeNames())) + continue + } + routes[Type(pr.Name)] = pr.Target + } + creds := credential.Store(roots.Home) + r := newRouter(routes, func(t oracle.Target) Judge { return Adapter{Calls: cfg, Creds: creds, Target: t} }, + func(p lab.Pair) (string, string, error) { return lab.RecordPair(repoRoot, p) }) + r.Diagnostics = diags + return r, nil +} + +// Mode is how t is routed: oracle.JudgementHost, JudgementShadow or +// JudgementOn. +func (r *Router) Mode(t Type) string { + if tgt, ok := r.routes[t]; ok && (tgt.Mode == oracle.JudgementShadow || tgt.Mode == oracle.JudgementOn) { + return tgt.Mode + } + return oracle.JudgementHost +} + +func (r *Router) window() int { + if r.Window > 0 { + return r.Window + } + return DefaultWindow +} + +// Decide answers q over s on the route t's configuration names, with host as +// the host's judgement. Its error is the host's (ErrHostPending among them) +// or the question's refusal; an adapter's failure is never an error: in shadow +// it is recorded in the pair, and turned on it is a fallback to host. +func (r *Router) Decide(ctx context.Context, s State, q Question, host Judge) (Answer, Receipt, error) { + if err := q.Check(); err != nil { + return Answer{}, Receipt{}, err + } + mode := r.Mode(q.Type) + size := tokens(render(s, q)) + rc := Receipt{Type: q.Type, Mode: mode, Route: RouteHost, StateSHA256: s.SHA256(), StateTokens: size} + tgt := r.routes[q.Type] + if mode != oracle.JudgementHost { + rc.Target = tgt.String() + } + switch mode { + case oracle.JudgementOn: + if size > r.window() { + rc.Fallback = fmt.Sprintf("the state is about %d tokens, more than %s's window of %d, so it was not sent and the host decided", + size, rc.Target, r.window()) + return r.byHost(ctx, s, q, host, rc) + } + a, err := r.ask(ctx, tgt, s, q) + if err != nil { + rc.Fallback = fmt.Sprintf("%s did not answer (%s), so the host decided", rc.Target, detail(err)) + return r.byHost(ctx, s, q, host, rc) + } + rc.Route, rc.ModelReported, rc.Probability = RouteAdapter, a.Model, a.Probability + return a, rc, nil + case oracle.JudgementShadow: + a, rc, err := r.byHost(ctx, s, q, host, rc) + if err != nil { + return a, rc, err + } + pair := lab.Pair{Type: string(q.Type), Kind: string(q.Kind), StateSHA256: rc.StateSHA256, StateTokens: size, + Host: a.Value, Target: rc.Target} + if size > r.window() { + pair.Skipped = lab.SkippedWindow + pair.Detail = fmt.Sprintf("the state is about %d tokens, more than the window of %d; the host decided alone", size, r.window()) + } else if sa, err := r.ask(ctx, tgt, s, q); err != nil { + pair.Skipped, pair.Detail = lab.SkippedFailed, detail(err) + } else { + p := sa.Probability + pair.Adapter, pair.Probability, pair.ModelReported = sa.Value, &p, sa.Model + } + rc.PairSkipped = pair.Skipped + if r.record == nil { + rc.Note = "the shadow pair was not kept: no recorder" + return a, rc, nil + } + if _, where, err := r.record(pair); err != nil { + rc.Note = "the shadow pair was not kept: " + detail(err) + } else { + rc.Pair = where + } + return a, rc, nil + default: + return r.byHost(ctx, s, q, host, rc) + } +} + +// byHost is the host's decision, on rc. +func (r *Router) byHost(ctx context.Context, s State, q Question, host Judge, rc Receipt) (Answer, Receipt, error) { + if host == nil { + return Answer{}, rc, ErrHostPending + } + a, err := host.Judge(ctx, s, q) + if err != nil { + return Answer{}, rc, err + } + rc.Route, rc.Probability = RouteHost, a.Probability + return a, rc, nil +} + +// ask is the adapter's answer, held to the question: a value among its +// choices and a probability from 0 to 1, whatever implementation answered. +func (r *Router) ask(ctx context.Context, tgt oracle.Target, s State, q Question) (Answer, error) { + if r.adapter == nil { + return Answer{}, errors.New("no adapter") + } + j := r.adapter(tgt) + if j == nil { + return Answer{}, errors.New("no adapter") + } + a, err := j.Judge(ctx, s, q) + if err != nil { + return Answer{}, err + } + v, ok := q.match(a.Value) + if !ok { + return Answer{}, fmt.Errorf("its answer %q is not one of %s", termsafe.Sanitize(clip(a.Value)), strings.Join(q.Choices(), ", ")) + } + if math.IsNaN(a.Probability) || a.Probability < 0 || a.Probability > 1 { + return Answer{}, fmt.Errorf("its probability %v is not from 0 to 1", a.Probability) + } + a.Value = v + return a, nil +} + +// detail is an error's account for a pair or a receipt: terminal-safe, one +// line, bounded. +func detail(err error) string { + s := strings.Join(strings.Fields(termsafe.Sanitize(err.Error())), " ") + if len(s) > maxDetail { + s = s[:maxDetail] + "…" + } + return s +} diff --git a/internal/core/implement/loop/drain.go b/internal/core/implement/loop/drain.go index d3dad30ae5..dc588c6cec 100644 --- a/internal/core/implement/loop/drain.go +++ b/internal/core/implement/loop/drain.go @@ -634,6 +634,10 @@ func drainMove(repoRoot string, o Options, d DrainOptions, flags paceFlags, rule st.Judging = nil res.Judged = &j pruneLapse(&st, j.Issue) + // With eligibility-residual in shadow, the decision adapter is asked + // beside the answer just taken and the pair is recorded in the newest + // lab home; it changes nothing the move does (drain_decide.go). + shadowDrainJudgement(d.Context, repoRoot, o, j, plan) return nil } if d.Judgement != "" { @@ -1026,22 +1030,31 @@ func drainMove(repoRoot string, o Options, d DrainOptions, flags paceFlags, rule judging.Hold = judgeHold st.Judging = &judging pruneLapse(&st, v.ID) - // A judge routed to a runner answers in this move, taken as the - // host's answer is; one that does not answer leaves the host's - // request standing, and is not started again for it - // (drain_judge.go). + // The decision adapter, when eligibility-residual is turned on, + // and then a judge routed to a runner answer in this move, taken + // as the host's answer is; one that does not answer leaves the + // host's request standing, and is not asked again for it + // (drain_decide.go, drain_judge.go). var taken *DrainJudgement clause := "" if judging.RoutedAsked { - clause = judgeAskedClause(v.ID) + clause = adapterAskedClause(repoRoot, o, d, v.ID) } else { - var started bool - if taken, clause, started, err = routedJudgement(repoRoot, iss, v, judging, plan, &st, o, d, now); err != nil { + var adapterAsked, started bool + if taken, clause, adapterAsked, err = adapterJudgement(d.Context, repoRoot, o, judging, plan, &st, now); err != nil { return refuseMoved(err) } - // Marked as asked only once the runner was started: a - // judge that never started is asked again next move. - if taken == nil && started { + if taken == nil { + var routed string + if taken, routed, started, err = routedJudgement(repoRoot, iss, v, judging, plan, &st, o, d, now); err != nil { + return refuseMoved(err) + } + clause = joinNext(clause, routed) + } + // Marked as asked only once the adapter was asked or the + // runner was started: a judge that never started is asked + // again next move. + if taken == nil && (adapterAsked || started) { st.Judging.RoutedAsked = true } } diff --git a/internal/core/implement/loop/drain_adapter_wire_test.go b/internal/core/implement/loop/drain_adapter_wire_test.go new file mode 100644 index 0000000000..2bae9c801b --- /dev/null +++ b/internal/core/implement/loop/drain_adapter_wire_test.go @@ -0,0 +1,214 @@ +package loop + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/intentdriven/abcd/internal/adapter/openaiapi" + "github.com/intentdriven/abcd/internal/core/lab" + "github.com/intentdriven/abcd/internal/core/oracle" + "github.com/intentdriven/abcd/internal/core/runner" +) + +// The drain's judgement through the decision adapter, as a drain move runs it +// (itd-2609221009495079 criteria 1, 3 and 5): the type eligibility-residual on +// the host leaves the drain as it was; in shadow the host's answer is taken +// and the adapter's is recorded beside it, or skipped with the state's size +// when the state is over the window; turned on, the adapter's answer is taken +// in the move that asks it, the run record naming the route, and an adapter +// that cannot answer leaves the host's request standing with a fallback +// receipt in the drain's state, asked once per request. + +// drainStateOf reads the drain's state as a move left it. +func drainStateOf(t *testing.T, root string) DrainState { + t.Helper() + raw, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(DrainStateRel))) + if err != nil { + t.Fatal(err) + } + var st DrainState + if err := json.Unmarshal(raw, &st); err != nil { + t.Fatal(err) + } + return st +} + +// TestADrainOnTheHostAsksNoAdapter: with eligibility-residual on the host, +// the move writes the host's request as it always did, the host's answer +// opens the lane, and the adapter is never asked and no pair is kept. +func TestADrainOnTheHostAsksNoAdapter(t *testing.T) { + repo := issueRepo(t) + adapter, got := &drainFakeJudge{value: HandBackTrustRule, p: 0.9}, []lab.Pair{} + withDrainRouter(t, oracle.JudgementHost, adapter, &got) + res, err := Drain(repo.Root(), Options{}, DrainOptions{}) + if err != nil { + t.Fatal(err) + } + if res.Judging == nil || res.Judged != nil || !strings.Contains(res.Next, "abcd drain --judgement") || strings.Contains(res.Next, drainAdapterRoute) { + t.Fatalf("the host's request is written and awaited: %+v", res) + } + res = judge(t, repo, Options{}, res, JudgementNo, "", "a renderer guard") + if res.Judged == nil || res.Judged.Route != nil || res.Start == nil { + t.Fatalf("the host's no is taken and opens the lane: %+v", res) + } + if adapter.calls != 0 || len(got) != 0 || len(drainStateOf(t, repo.Root()).JudgeFallbacks) != 0 { + t.Fatalf("on the host the adapter is asked %d times, %d pairs are kept", adapter.calls, len(got)) + } +} + +// TestADrainInShadowRecordsThePairAndKeepsTheHostsAnswer is criterion 1 in +// the drain: the host's answer decides, and once it is taken the adapter is +// asked the same question and the pair is recorded; the adapter's different +// answer changes nothing. +func TestADrainInShadowRecordsThePairAndKeepsTheHostsAnswer(t *testing.T) { + repo := issueRepo(t) + adapter, got := &drainFakeJudge{value: HandBackTrustRule, p: 0.8}, []lab.Pair{} + withDrainRouter(t, oracle.JudgementShadow, adapter, &got) + res, err := Drain(repo.Root(), Options{}, DrainOptions{}) + if err != nil { + t.Fatal(err) + } + if res.Judging == nil || adapter.calls != 0 { + t.Fatalf("shadow asks the adapter nothing before the host answers: %+v, calls %d", res, adapter.calls) + } + res = judge(t, repo, Options{}, res, JudgementNo, "", "a renderer guard") + if res.Judged == nil || res.Judged.Answer != JudgementNo || res.Judged.Route != nil || res.Start == nil { + t.Fatalf("the host's no is taken and opens the lane, whatever the adapter said: %+v", res) + } + if adapter.calls != 1 || len(got) != 1 { + t.Fatalf("the adapter is asked once beside the host and one pair kept: calls %d, pairs %+v", adapter.calls, got) + } + p := got[0] + if p.Type != "eligibility-residual" || p.Host != JudgementNo || p.Adapter != HandBackTrustRule || p.Probability == nil || + *p.Probability != 0.8 || len(p.StateSHA256) != 64 || p.Skipped != "" { + t.Fatalf("pair = %+v", p) + } + if st := drainStateOf(t, repo.Root()); len(st.JudgeFallbacks) != 0 { + t.Fatalf("shadow keeps no fallback in the drain's state: %+v", st.JudgeFallbacks) + } +} + +// TestADrainInShadowSkipsAStateOverTheWindow is criterion 5 in the drain: a +// state larger than the adapter's window is not sent; the pair is recorded as +// skipped with the state's size, and the host's answer decides alone. +func TestADrainInShadowSkipsAStateOverTheWindow(t *testing.T) { + repo := issueRepo(t) + adapter, got := &drainFakeJudge{value: JudgementNo, p: 0.9}, []lab.Pair{} + withDrainRouterWindow(t, oracle.JudgementShadow, 8, adapter, &got) + res, err := Drain(repo.Root(), Options{}, DrainOptions{}) + if err != nil { + t.Fatal(err) + } + res = judge(t, repo, Options{}, res, JudgementNo, "", "a renderer guard") + if res.Judged == nil || res.Start == nil { + t.Fatalf("the host's no opens the lane: %+v", res) + } + if adapter.calls != 0 || len(got) != 1 { + t.Fatalf("a state over the window is not sent: calls %d, pairs %+v", adapter.calls, got) + } + if p := got[0]; p.Skipped != lab.SkippedWindow || p.StateTokens <= 8 || p.Host != JudgementNo || + !strings.Contains(p.Detail, "tokens") || p.Adapter != "" { + t.Fatalf("pair = %+v, want a window skip with the size", p) + } +} + +// TestADrainTurnedOnTakesTheAdaptersAnswerAndNamesTheRoute is criterion 3's +// first half in the drain: the adapter's no is taken in the move that asks +// it, the lane opens, and the drain's record names the adapter's route. +func TestADrainTurnedOnTakesTheAdaptersAnswerAndNamesTheRoute(t *testing.T) { + repo := issueRepo(t) + adapter, got := &drainFakeJudge{value: JudgementNo, p: 0.97}, []lab.Pair{} + withDrainRouter(t, oracle.JudgementOn, adapter, &got) + res, err := Drain(repo.Root(), Options{}, DrainOptions{}) + if err != nil { + t.Fatal(err) + } + if res.Judging != nil || res.Judged == nil || res.Judged.Answer != JudgementNo || !res.Judged.Applied || res.Start == nil { + t.Fatalf("the adapter's no is taken and opens the lane in the same move: %+v", res) + } + want := drainAdapterRoute + " openrouter/typesafe/jev-1.13" + st := drainStateOf(t, repo.Root()) + if len(st.Judgements) != 1 { + t.Fatalf("judgements = %+v", st.Judgements) + } + if r := st.Judgements[0].Route; r == nil || r.Asked != want || r.Ran != want || r.Model != "typesafe/jev-1.13-0901" { + t.Fatalf("the drain's record names the route: %+v", r) + } + if adapter.calls != 1 || len(got) != 0 || len(st.JudgeFallbacks) != 0 { + t.Fatalf("calls %d, pairs %d, fallbacks %+v", adapter.calls, len(got), st.JudgeFallbacks) + } +} + +// TestADrainTurnedOnFallsBackToTheHostWithAReceiptOnce is criterion 3's +// second half in the drain: an adapter that cannot answer leaves the host's +// request standing, the fallback receipt kept in the drain's state; the polls +// that follow while the request stands ask it nothing again, and the host's +// answer is taken as it always was. +func TestADrainTurnedOnFallsBackToTheHostWithAReceiptOnce(t *testing.T) { + repo := issueRepo(t) + adapter, got := &drainFakeJudge{err: openaiapi.ErrUnreachable}, []lab.Pair{} + withDrainRouter(t, oracle.JudgementOn, adapter, &got) + res, err := Drain(repo.Root(), Options{}, DrainOptions{}) + if err != nil { + t.Fatal(err) + } + if res.Judging == nil || res.Judged != nil || res.Start != nil { + t.Fatalf("the host's request stands: %+v", res) + } + if !strings.Contains(res.Next, "abcd drain --judgement") || !strings.Contains(res.Next, "the decision adapter did not judge "+eligibleIssue) { + t.Fatalf("next names the fallback and the host's answer: %s", res.Next) + } + fbs := drainStateOf(t, repo.Root()).JudgeFallbacks + if len(fbs) != 1 || fbs[0].Role != RoleDrainJudge || fbs[0].Ran != runner.Host || + fbs[0].Asked != drainAdapterRoute+" openrouter/typesafe/jev-1.13" || !strings.Contains(fbs[0].Detail, "the host's request stands") { + t.Fatalf("the fallback receipt is kept in the drain's state: %+v", fbs) + } + for range 2 { + poll, err := Drain(repo.Root(), Options{}, DrainOptions{}) + if err != nil { + t.Fatal(err) + } + if poll.Judging == nil || poll.Judged != nil || !strings.Contains(poll.Next, "the decision adapter was already asked over "+eligibleIssue) { + t.Fatalf("the host's request still stands, and the adapter is not asked again: %+v", poll) + } + res = poll + } + if adapter.calls != 1 || len(drainStateOf(t, repo.Root()).JudgeFallbacks) != 1 { + t.Fatalf("the adapter is asked once per request: %d calls, fallbacks %+v", adapter.calls, drainStateOf(t, repo.Root()).JudgeFallbacks) + } + res = judge(t, repo, Options{}, res, JudgementNo, "", "a renderer guard") + if res.Judged == nil || res.Judged.Route != nil || res.Start == nil { + t.Fatalf("the host's answer is taken and opens the lane: %+v", res) + } +} + +// TestADrainStoppedWhileTheAdapterJudgesTakesNoAnswer: a drain stopped while +// the decision adapter is asked refuses the move, records no fallback for the +// stop, and leaves the request unasked, so the next move asks again. +func TestADrainStoppedWhileTheAdapterJudgesTakesNoAnswer(t *testing.T) { + repo := issueRepo(t) + adapter, got := &drainFakeJudge{err: context.Canceled}, []lab.Pair{} + withDrainRouter(t, oracle.JudgementOn, adapter, &got) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := Drain(repo.Root(), Options{}, DrainOptions{Context: ctx}); err == nil || !strings.Contains(err.Error(), "stopped while the decision adapter judged") { + t.Fatalf("a stopped drain refuses the move: %v", err) + } + // A first move refused writes no state at all; any it wrote keeps no + // fallback and leaves the request unasked. + if _, err := os.Stat(filepath.Join(repo.Root(), filepath.FromSlash(DrainStateRel))); err == nil { + st := drainStateOf(t, repo.Root()) + if len(st.JudgeFallbacks) != 0 || (st.Judging != nil && st.Judging.RoutedAsked) { + t.Fatalf("the stop is no fallback and the request is asked again: %+v", st) + } + } + adapter.err, adapter.value, adapter.p = nil, JudgementNo, 0.9 + res, err := Drain(repo.Root(), Options{}, DrainOptions{}) + if err != nil || res.Judged == nil || res.Judged.Route == nil || adapter.calls != 2 { + t.Fatalf("the next move asks the adapter again and takes its answer: %+v, %v, calls %d", res, err, adapter.calls) + } +} diff --git a/internal/core/implement/loop/drain_decide.go b/internal/core/implement/loop/drain_decide.go new file mode 100644 index 0000000000..2d4dd2fd91 --- /dev/null +++ b/internal/core/implement/loop/drain_decide.go @@ -0,0 +1,218 @@ +package loop + +// drain_decide.go is the decision adapter at the drain's judgement +// (itd-2609221009495079, spc-2609221011151661 scope 2). The judgement a drain +// asks before it opens an issue's lane is a closed-option question over one +// record — no, or yes and which kind of hand-back — so it is the judgement +// type eligibility-residual: the residual of the drain's eligibility once the +// field rule has passed, and it is routed through core/decide/judge, as every +// typed judgement is. +// +// - Configured for shadow, the host (or the routed runner) decides exactly +// as it does today; once its answer is taken, the adapter is asked the +// same question over the same record and the pair is recorded in the +// newest lab home (shadowDrainJudgement). Shadow never changes or stops +// the drain, and its one call, made after the answer is taken, is bounded +// by drainShadowTimeout. +// - Turned on, the adapter is asked once the host's request is written +// (decideDrainJudgement, through adapterJudgement), before any runner the +// drain's judge is routed to: its answer is taken as a routed judge's is, +// under the same disposition rules (judgeAnswer), and the judgement's +// Route names the adapter and the model that answered. An adapter that +// cannot answer leaves the host's request standing, with a fallback +// receipt kept beside the routed judge's, and the request is marked as +// asked, so the polls that follow ask it nothing again. +// +// Both functions read the configuration themselves (loadDrainRouter), so the +// drain's call sites pass only what they already hold. On the host, the +// default, neither asks anything, and the drain moves as it did before the +// adapter existed. + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/intentdriven/abcd/internal/core/capture" + decision "github.com/intentdriven/abcd/internal/core/decide/judge" + "github.com/intentdriven/abcd/internal/core/oracle" + "github.com/intentdriven/abcd/internal/core/runner" + "github.com/intentdriven/abcd/internal/termsafe" +) + +// drainJudgementType is the drain's judgement in the decision adapter's +// registry. +const drainJudgementType = decision.EligibilityResidual + +// drainAdapterRoute names the decision adapter in a judgement's Route and a +// fallback receipt's Asked, beside its /. +const drainAdapterRoute = "decision-adapter" + +// drainAdapterTimeout bounds one adapter call: a typed answer over one record +// takes about a second, so this is for a provider that hangs. +const drainAdapterTimeout = 2 * time.Minute + +// drainShadowTimeout bounds the adapter's call in shadow, where its answer +// decides nothing: a provider slower than this costs the pair (recorded as +// failed), never the move's time. +const drainShadowTimeout = 20 * time.Second + +// loadDrainRouter reads the decision adapter's configuration for the checkout +// at repoRoot. Tests replace it. +var loadDrainRouter = func(repoRoot string, o Options) (*decision.Router, error) { + return decision.Load(o.roots(repoRoot), repoRoot) +} + +// drainQuestion is the drain's one question as a typed judgement: no, or the +// kind of hand-back a yes names. +func drainQuestion() decision.Question { + return decision.Question{Type: drainJudgementType, Kind: decision.KindChoose, + Ask: "Does this remedy, carried out as written, change what a user sees, or a trust boundary? " + + "What a user sees is anything a person reads or relies on: a command's output or exit code, a flag, " + + "a message, a page, a prompt, or what a command does when a person runs it. A trust boundary is what is " + + "allowed or refused, read, written, run or sent, whose input is trusted, or a rule about safety, secrets " + + "or permissions. Answer " + HandBackUserVisible + " when it changes what a user sees, " + HandBackTrustRule + + " when it changes a trust boundary (or both, or you cannot tell), and no only when it changes neither.", + Options: []string{JudgementNo, HandBackUserVisible, HandBackTrustRule}} +} + +// drainState is the state the judgement is made over: the issue as the +// host's request shows it, without the request's instructions about where to +// write the answer. +func drainState(iss capture.Issue) decision.State { + var b strings.Builder + fmt.Fprintf(&b, "Issue %s\nseverity: %s\ncategory: %s\n\nRemedy:\n%s\n\nRecord body:\n%s\n", + iss.ID, termsafe.Sanitize(string(iss.Severity)), termsafe.Sanitize(string(iss.Category)), + termsafe.Sanitize(iss.Remedy), termsafe.SanitizeBlock(strings.TrimSpace(iss.Body))) + return decision.State{Text: b.String()} +} + +// drainValue is a taken judgement as the question's choice: no, or the kind. +func drainValue(j DrainJudgement) string { + if j.Answer == JudgementNo { + return JudgementNo + } + return j.Kind +} + +// shadowDrainJudgement asks the adapter beside a judgement the drain has +// taken, j, when eligibility-residual is configured for shadow, and records +// the pair. It returns the receipt, nil when the type is not in shadow, the +// issue has left the open ledger, or the configuration cannot be read (the +// drain has its answer; shadow never stops it). The drain's take point calls +// it once j is applied and recorded, so an answer the move could not take is +// never paired. +func shadowDrainJudgement(ctx context.Context, repoRoot string, o Options, j DrainJudgement, plan capture.DrainPlan) *decision.Receipt { + iss, ok := plan.OpenIssue(j.Issue) + if !ok || drainValue(j) == "" { + return nil + } + r, err := loadDrainRouter(repoRoot, o) + if err != nil || r.Mode(drainJudgementType) != oracle.JudgementShadow { + return nil + } + ctx, cancel := context.WithTimeout(ctxOr(ctx), drainShadowTimeout) + defer cancel() + _, rc, err := r.Decide(ctx, drainState(iss), drainQuestion(), decision.Answered(drainValue(j))) + if err != nil { + return nil + } + return &rc +} + +// decideDrainJudgement asks the adapter for the judgement the drain awaits, +// judging, when eligibility-residual is turned on. It returns the judgement +// the adapter decided, taken as a routed judge's answer is (judgeAnswer), with +// its Route naming the adapter; or nil and the fallback receipt when the +// adapter could not answer, the host's request then standing; or nil, nil when +// the type is not turned on. A configuration the oracle reader refuses (a +// model off its provider's list) is an error, since a route the person set is +// never dropped silently; nothing was called. +func decideDrainJudgement(ctx context.Context, repoRoot string, o Options, judging DrainJudging, plan capture.DrainPlan, + now time.Time) (*DrainJudgement, *runner.FallbackReceipt, error) { + iss, ok := plan.OpenIssue(judging.Issue) + if !ok { + return nil, nil, nil + } + r, err := loadDrainRouter(repoRoot, o) + if err != nil { + return nil, nil, fmt.Errorf("the decision adapter's configuration: %w", err) + } + if r.Mode(drainJudgementType) != oracle.JudgementOn { + return nil, nil, nil + } + ctx, cancel := context.WithTimeout(ctxOr(ctx), drainAdapterTimeout) + defer cancel() + a, rc, err := r.Decide(ctx, drainState(iss), drainQuestion(), decision.Pending()) + asked := drainAdapterRoute + " " + rc.Target + if errors.Is(err, decision.ErrHostPending) { + return nil, &runner.FallbackReceipt{At: now, Role: RoleDrainJudge, Asked: asked, Reason: runner.ReasonFailed, + Detail: strings.TrimSuffix(rc.Fallback, "the host decided") + "the host's request stands", Ran: runner.Host}, nil + } + if err != nil { + return nil, nil, err + } + ans := JudgementAnswer{SchemaVersion: DrainJudgementSchemaVersion, Issue: judging.Issue, RemedySHA256: judging.RemedySHA256, + Answer: JudgementYes, Kind: a.Value, + Reason: fmt.Sprintf("the decision adapter %s answered %s at probability %.2f", rc.Target, a.Value, a.Probability)} + if a.Value == JudgementNo { + ans.Answer, ans.Kind = JudgementNo, "" + } + taken, err := judgeAnswer(ans, judging, plan, now) + if err != nil { + return nil, nil, err + } + taken.Route = &runner.RouteRecord{Asked: asked, Ran: asked, Model: termsafe.Sanitize(rc.ModelReported)} + return &taken, nil, nil +} + +// adapterJudgement is the drain's move over the request it awaits, judging, +// when eligibility-residual is turned on: the adapter's answer, taken as a +// routed judge's is, or nil with the clause the move's next step adds and the +// fallback receipt kept in st, the host's request then standing. asked +// reports whether the adapter was asked and did not answer, so the request is +// marked as asked (DrainJudging.RoutedAsked) and the polls that follow while +// the host's request stands ask it nothing again. Not turned on, it asks +// nothing and returns nil, "", false. A drain stopped while the adapter was +// asked is no fallback: the move refuses, and the next one asks again. +func adapterJudgement(ctx context.Context, repoRoot string, o Options, judging DrainJudging, plan capture.DrainPlan, + st *DrainState, now time.Time) (taken *DrainJudgement, clause string, asked bool, err error) { + ctx = ctxOr(ctx) + taken, fb, err := decideDrainJudgement(ctx, repoRoot, o, judging, plan, now) + if err != nil { + return nil, "", false, err + } + if fb == nil { + return taken, "", false, nil + } + if ctx.Err() != nil { + return nil, "", false, refuse(StageDrain, "", "", "the drain was stopped while the decision adapter judged "+judging.Issue+ + "; the move took no answer", "run `abcd drain` again, which asks the decision adapter again") + } + keepJudgeFallback(st, *fb) + return nil, "the decision adapter did not judge " + judging.Issue + ", recorded as a fallback: " + fb.Detail, true, nil +} + +// adapterAskedClause is the move's clause for a request marked as asked +// (DrainJudging.RoutedAsked) when the drain's judge is not routed to a +// runner and eligibility-residual is turned on: the decision adapter was the +// one asked, in an earlier move, and is not asked again. Otherwise it is the +// routed runner's clause (judgeAskedClause). +func adapterAskedClause(repoRoot string, o Options, d DrainOptions, issue string) string { + if !judgeRouted(o, d) { + if r, err := loadDrainRouter(repoRoot, o); err == nil && r.Mode(drainJudgementType) == oracle.JudgementOn { + return "the decision adapter was already asked over " + issue + " for this request and did not answer, so it is not asked again and the request is the host's" + } + } + return judgeAskedClause(issue) +} + +// ctxOr is ctx, or the background context when it is nil. +func ctxOr(ctx context.Context) context.Context { + if ctx == nil { + return context.Background() + } + return ctx +} diff --git a/internal/core/implement/loop/drain_decide_test.go b/internal/core/implement/loop/drain_decide_test.go new file mode 100644 index 0000000000..4e538eee90 --- /dev/null +++ b/internal/core/implement/loop/drain_decide_test.go @@ -0,0 +1,183 @@ +package loop + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/intentdriven/abcd/internal/abcdhome" + "github.com/intentdriven/abcd/internal/adapter/openaiapi" + "github.com/intentdriven/abcd/internal/core/capture" + decision "github.com/intentdriven/abcd/internal/core/decide/judge" + "github.com/intentdriven/abcd/internal/core/lab" + "github.com/intentdriven/abcd/internal/core/layered" + "github.com/intentdriven/abcd/internal/core/oracle" + "github.com/intentdriven/abcd/internal/core/runner" +) + +// The drain's judgement through the decision adapter (itd-2609221009495079): +// eligibility-residual in shadow records a pair beside the host's answer and +// changes nothing; turned on, the adapter's answer is taken as a routed +// judge's is, and one it cannot give leaves the host's request standing with +// a fallback receipt. + +type drainFakeJudge struct { + value string + p float64 + err error + calls int +} + +func (f *drainFakeJudge) Judge(context.Context, decision.State, decision.Question) (decision.Answer, error) { + f.calls++ + if f.err != nil { + return decision.Answer{}, f.err + } + return decision.Answer{Value: f.value, Probability: f.p, Model: "typesafe/jev-1.13-0901"}, nil +} + +// withDrainRouter makes the drain judge with eligibility-residual in mode, +// adapter answering, and pairs kept in got. +func withDrainRouter(t *testing.T, mode string, adapter *drainFakeJudge, got *[]lab.Pair) { + t.Helper() + withDrainRouterWindow(t, mode, 0, adapter, got) +} + +// withDrainRouterWindow is withDrainRouter with the adapter's window in +// tokens; the default window when zero. +func withDrainRouterWindow(t *testing.T, mode string, window int, adapter *drainFakeJudge, got *[]lab.Pair) { + t.Helper() + routes := map[decision.Type]oracle.Target{} + if mode != oracle.JudgementHost { + routes[decision.EligibilityResidual] = oracle.Target{Provider: "openrouter", Model: "typesafe/jev-1.13", Mode: mode} + } + prev := loadDrainRouter + loadDrainRouter = func(string, Options) (*decision.Router, error) { + r := decision.NewForTest(routes, func(oracle.Target) decision.Judge { return adapter }, func(p lab.Pair) (string, string, error) { + *got = append(*got, p) + return "lab-261029000000-abcdef0", abcdhome.Display("lab/x/lab-261029000000-abcdef0/pairs.jsonl"), nil + }) + r.Window = window + return r, nil + } + t.Cleanup(func() { loadDrainRouter = prev }) +} + +func drainDecidePlan(t *testing.T) (*capture.DrainPlan, string, DrainJudging) { + t.Helper() + repo := drainRepo(t) + plan, err := capture.PlanDrain(capture.DrainPlanRequest{RepoRoot: repo.Root()}) + if err != nil { + t.Fatal(err) + } + iss, ok := plan.OpenIssue(eligibleIssue) + if !ok { + t.Fatalf("%s is not open in the plan", eligibleIssue) + } + return &plan, repo.Root(), DrainJudging{Issue: eligibleIssue, RemedySHA256: remedyDigest(iss.Remedy)} +} + +func TestShadowDrainJudgementRecordsThePairBesideTheHostsAnswer(t *testing.T) { + plan, root, _ := drainDecidePlan(t) + adapter, got := &drainFakeJudge{value: HandBackTrustRule, p: 0.8}, []lab.Pair{} + withDrainRouter(t, oracle.JudgementShadow, adapter, &got) + j := DrainJudgement{Issue: eligibleIssue, Answer: JudgementNo, Reason: "a renderer guard", Applied: true} + rc := shadowDrainJudgement(context.Background(), root, Options{}, j, *plan) + if rc == nil || rc.Route != decision.RouteHost || rc.Type != decision.EligibilityResidual || !strings.HasSuffix(rc.Pair, "pairs.jsonl") { + t.Fatalf("receipt = %+v", rc) + } + if len(got) != 1 || got[0].Host != JudgementNo || got[0].Adapter != HandBackTrustRule || *got[0].Probability != 0.8 || + got[0].Type != string(decision.EligibilityResidual) { + t.Fatalf("pairs = %+v", got) + } + // A yes is paired as its kind. + j = DrainJudgement{Issue: eligibleIssue, Answer: JudgementYes, Kind: HandBackUserVisible, Reason: "a flag"} + shadowDrainJudgement(context.Background(), root, Options{}, j, *plan) + if len(got) != 2 || got[1].Host != HandBackUserVisible { + t.Fatalf("pairs = %+v", got) + } +} + +func TestShadowDrainJudgementDoesNothingOffShadow(t *testing.T) { + plan, root, _ := drainDecidePlan(t) + for _, mode := range []string{oracle.JudgementHost, oracle.JudgementOn} { + adapter, got := &drainFakeJudge{value: JudgementNo, p: 0.9}, []lab.Pair{} + withDrainRouter(t, mode, adapter, &got) + if rc := shadowDrainJudgement(context.Background(), root, Options{}, DrainJudgement{Issue: eligibleIssue, Answer: JudgementNo}, *plan); rc != nil || adapter.calls != 0 || len(got) != 0 { + t.Fatalf("%s: receipt %+v, adapter calls %d, pairs %d", mode, rc, adapter.calls, len(got)) + } + } +} + +func TestDecideDrainJudgementTakesTheAdaptersAnswerWhenTurnedOn(t *testing.T) { + plan, root, judging := drainDecidePlan(t) + adapter, got := &drainFakeJudge{value: HandBackUserVisible, p: 0.91}, []lab.Pair{} + withDrainRouter(t, oracle.JudgementOn, adapter, &got) + now := time.Date(2026, 10, 29, 6, 0, 0, 0, time.UTC) + taken, fb, err := decideDrainJudgement(context.Background(), root, Options{}, judging, *plan, now) + if err != nil || fb != nil || taken == nil { + t.Fatalf("taken %+v, fallback %+v, err %v", taken, fb, err) + } + if taken.Answer != JudgementYes || taken.Kind != HandBackUserVisible || !taken.Applied || !taken.At.Equal(now) || + !strings.Contains(taken.Reason, "probability 0.91") { + t.Fatalf("taken = %+v", taken) + } + if r := taken.Route; r == nil || r.Asked != "decision-adapter openrouter/typesafe/jev-1.13" || r.Ran != r.Asked || r.Model != "typesafe/jev-1.13-0901" { + t.Fatalf("route = %+v: the run record names the route", taken.Route) + } + // A no is taken as a no, and opens the lane. + adapter.value = JudgementNo + taken, _, _ = decideDrainJudgement(context.Background(), root, Options{}, judging, *plan, now) + if taken == nil || taken.Answer != JudgementNo || taken.Kind != "" || !taken.Applied { + t.Fatalf("no = %+v", taken) + } +} + +func TestDecideDrainJudgementFallsBackToTheHostWithAReceipt(t *testing.T) { + plan, root, judging := drainDecidePlan(t) + adapter, got := &drainFakeJudge{err: openaiapi.ErrUnreachable}, []lab.Pair{} + withDrainRouter(t, oracle.JudgementOn, adapter, &got) + taken, fb, err := decideDrainJudgement(context.Background(), root, Options{}, judging, *plan, time.Now()) + if err != nil || taken != nil || fb == nil { + t.Fatalf("taken %+v, fallback %+v, err %v", taken, fb, err) + } + if fb.Role != RoleDrainJudge || fb.Ran != runner.Host || !strings.Contains(fb.Asked, "openrouter/typesafe/jev-1.13") || + !strings.Contains(fb.Detail, "the host's request stands") { + t.Fatalf("fallback = %+v", fb) + } +} + +func TestDecideDrainJudgementAsksNothingUnlessTurnedOn(t *testing.T) { + plan, root, judging := drainDecidePlan(t) + for _, mode := range []string{oracle.JudgementHost, oracle.JudgementShadow} { + adapter, got := &drainFakeJudge{value: JudgementNo, p: 0.9}, []lab.Pair{} + withDrainRouter(t, mode, adapter, &got) + taken, fb, err := decideDrainJudgement(context.Background(), root, Options{}, judging, *plan, time.Now()) + if taken != nil || fb != nil || err != nil || adapter.calls != 0 { + t.Fatalf("%s: taken %+v fallback %+v err %v calls %d", mode, taken, fb, err, adapter.calls) + } + } +} + +// The real configuration: a model off its provider's list refuses the +// judgement before any call, naming the list. +func TestDecideDrainJudgementRefusesAnUnlistedModelBeforeAnyCall(t *testing.T) { + plan, root, judging := drainDecidePlan(t) + home := t.TempDir() + cfg := abcdhome.Path(home, "config.json") + if err := os.MkdirAll(filepath.Dir(cfg), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(cfg, []byte(`{"oracle":{"api":{"openrouter":{"base_url":"https://openrouter.ai/api/v1","key":"openrouter", + "models":["typesafe/jev-1.13"]}},"judgements":{"eligibility-residual":"openrouter/mistral/small"}}}`), 0o600); err != nil { + t.Fatal(err) + } + o := Options{Roots: &layered.Roots{Repo: root, Home: home}} + _, _, err := decideDrainJudgement(context.Background(), root, o, judging, *plan, time.Now()) + if err == nil || !strings.Contains(err.Error(), "not on openrouter's list") { + t.Fatalf("err = %v, want the allowlist refusal", err) + } +} diff --git a/internal/core/lab/harvest.go b/internal/core/lab/harvest.go index 716cf7f2eb..c0c2e5ec9e 100644 --- a/internal/core/lab/harvest.go +++ b/internal/core/lab/harvest.go @@ -51,13 +51,20 @@ type Harvested struct { Gaps []Gap `json:"gaps"` Coverage []SectionCoverage `json:"coverage"` Halted []string `json:"halted"` + // Agreement is the decision adapter's shadow agreement over the lab's + // pairs.jsonl (pairs.go); absent when the lab recorded no pair. + Agreement *Agreement `json:"agreement,omitempty"` } // Harvest assembles the lab's harvest in the lifeboat's section shape — // intention, method, what worked, what is open, candidates, coverage — from the // INTENTION, the findings log and the probe records, citing each finding's probe // records by path. Its product findings are listed as capture candidates, each -// with the capture line that files it, found during this lab. +// with the capture line that files it, found during this lab. A lab that +// recorded the decision adapter's shadow pairs (pairs.go) also gets the +// agreement per judgement type, with counts and probability bands, and the +// line a research note filed from it carries, naming the lab home and the +// count (itd-2609221009495079 criterion 2). // // No claim outlives its input: a finding that cites no probe record or evidence // file, or cites one that is missing or incomplete, is a gap, and a harvest with @@ -132,6 +139,9 @@ func Harvest(repoRoot, id string) (Harvested, error) { } } res.Coverage = coverage(intention, findings, probes, res) + if res.Agreement, err = l.agreement(); err != nil { + return Harvested{}, err + } if len(res.Gaps) > 0 { return res, fmt.Errorf("%w: %d finding citation(s) cannot be verified from the lab's records; nothing was written", ErrHalted, len(res.Gaps)) } @@ -301,5 +311,8 @@ func harvestDoc(e Entry, intention string, findings []Finding, probes map[string fmt.Fprintf(&b, "| %s | %s | %s |\n", c.Section, c.Status, c.Note) } b.WriteString("\nCost: this harvest records what the lab's records hold and claims no token or\nmoney cost the lab did not measure.\n") + if res.Agreement != nil { + b.WriteString("\n" + agreementDoc(res.Agreement)) + } return b.String() } diff --git a/internal/core/lab/pairs.go b/internal/core/lab/pairs.go new file mode 100644 index 0000000000..e42c50f20f --- /dev/null +++ b/internal/core/lab/pairs.go @@ -0,0 +1,313 @@ +package lab + +// pairs.go is the decision adapter's shadow record (itd-2609221009495079, +// spc-2609221011151661 scopes 3 and 4): while a judgement type runs in shadow +// the host decides and the adapter answers beside it, and each pair (the +// host's answer, the adapter's, its probability and the state's hash) is +// appended to pairs.jsonl in a lab home, in the machine-scoped store and never +// the repository. The harvest reads the file back as the agreement report a +// research note turns a type on with. +// +// A pair goes to the newest lab minted for the repository: a lab is minted to +// answer one question, and the shadow run is the question "does the adapter +// agree with the host?", so the person mints the lab before configuring a type +// for shadow, and the lab the latest mint opened collects the pairs. + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "sort" + "strings" + "syscall" + + "github.com/intentdriven/abcd/internal/fsutil" +) + +// pairsName is the shadow record inside a lab home. +const pairsName = "pairs.jsonl" + +// maxPairBytes bounds one recorded line: a pair carries answers and a hash, +// never the state itself. +const maxPairBytes = 8 << 10 + +// SkippedWindow marks a pair whose state exceeded the adapter's window, so +// only the host answered; SkippedFailed one whose adapter call failed. +const ( + SkippedWindow = "window" + SkippedFailed = "adapter-failed" +) + +// ErrNoLab is RecordPair's answer when the repository has no lab to record +// in: the host has decided, and the pair is not kept. +var ErrNoLab = errors.New("no lab minted for this repository") + +// Pair is one shadow judgement: the type and its question kind, the state's +// hash and size, the host's answer, and the adapter's answer with its +// probability, or why the adapter did not answer. +type Pair struct { + At string `json:"at"` + Type string `json:"type"` + Kind string `json:"kind"` + StateSHA256 string `json:"state_sha256"` + StateTokens int `json:"state_tokens"` + Host string `json:"host"` + Adapter string `json:"adapter,omitempty"` + Probability *float64 `json:"probability,omitempty"` + // Target is the adapter's /; ModelReported the model + // the provider said answered. + Target string `json:"target"` + ModelReported string `json:"model_reported,omitempty"` + // Skipped is SkippedWindow or SkippedFailed when the adapter gave no + // answer, and Detail says why. + Skipped string `json:"skipped,omitempty"` + Detail string `json:"detail,omitempty"` +} + +// RecordPair appends p to pairs.jsonl in the newest lab minted for the +// repository at repoRoot, stamping its time, and returns the lab's id and the +// file in tilde form. It refuses a pair without its type, kind, state hash or +// host answer, and returns ErrNoLab when the repository has no lab. +func RecordPair(repoRoot string, p Pair) (id, where string, err error) { + if p.Type == "" || p.Kind == "" || len(p.StateSHA256) != 64 || p.Host == "" { + return "", "", fmt.Errorf("%w: a pair names its type, kind, state hash and the host's answer", ErrRefused) + } + s, err := resolveStore(repoRoot) + if err != nil { + return "", "", err + } + es, _, err := s.entries() + if err != nil { + return "", "", err + } + if len(es) == 0 { + return "", "", fmt.Errorf("%w in %s; mint one with `abcd lab mint` to record shadow pairs", ErrNoLab, s.display()) + } + newest := es[0] + for _, e := range es[1:] { + if e.ID > newest.ID { + newest = e + } + } + l, err := open(repoRoot, newest.ID) + if err != nil { + return "", "", err + } + defer l.close() + p.At = stamp() + line, err := json.Marshal(p) + if err != nil { + return "", "", err + } + if len(line) >= maxPairBytes { + return "", "", fmt.Errorf("%w: a pair of %d bytes exceeds the %d a line may hold", ErrRefused, len(line), maxPairBytes) + } + if fi, err := l.root.Lstat(pairsName); err == nil && !fi.Mode().IsRegular() { + return "", "", fmt.Errorf("%w: %s is not a regular file", ErrRefused, l.display(pairsName)) + } + f, err := l.root.OpenFile(pairsName, os.O_WRONLY|os.O_APPEND|os.O_CREATE, fileMode) + if err != nil { + return "", "", fmt.Errorf("cannot open %s: %v", l.display(pairsName), redact(err, l.store.home)) + } + if _, err := f.Write(append(line, '\n')); err != nil { + f.Close() + return "", "", fmt.Errorf("cannot append to %s: %v", l.display(pairsName), redact(err, l.store.home)) + } + if err := f.Close(); err != nil { + return "", "", fmt.Errorf("cannot close %s: %v", l.display(pairsName), redact(err, l.store.home)) + } + return newest.ID, l.display(pairsName), nil +} + +// Band is one probability band of a type's compared pairs. +type Band struct { + Band string `json:"band"` + Compared int `json:"compared"` + Agreed int `json:"agreed"` +} + +// TypeAgreement is one judgement type's agreement: every pair recorded, the +// ones the adapter answered (compared), those whose answers matched, those it +// did not answer (skipped), the rate over the compared, and the bands. +type TypeAgreement struct { + Type string `json:"type"` + Pairs int `json:"pairs"` + Compared int `json:"compared"` + Agreed int `json:"agreed"` + Skipped int `json:"skipped"` + Rate float64 `json:"rate"` + Bands []Band `json:"bands"` +} + +// Agreement is the harvest's report over a lab's pairs: the lab home, the +// pairs counted, the lines that did not parse, each type's agreement, and the +// line a research note filed from it carries. +type Agreement struct { + LabHome string `json:"lab_home"` + Pairs int `json:"pairs"` + Unreadable int `json:"unreadable"` + Types []TypeAgreement `json:"types"` + Note string `json:"note"` +} + +// bands are the probability bands, lowest first: [lo, hi), the last closed. +var bands = []struct { + name string + lo, hi float64 +}{{"0.00-0.50", 0, 0.5}, {"0.50-0.70", 0.5, 0.7}, {"0.70-0.90", 0.7, 0.9}, {"0.90-1.00", 0.9, 1.0000001}} + +// agreement reads the lab's pairs.jsonl; nil when the lab recorded none. +// +// The file is read a line at a time, each line bounded by maxPairBytes, and +// never whole: RecordPair appends to it for as long as a type runs in shadow, +// so a busy lab's file outgrows any bound one document read could hold, and +// the harvest still counts every pair. A line longer than a pair may be is +// counted as one that did not parse, and the read goes on past it. +func (l *lab) agreement() (*Agreement, error) { + f, err := l.openPairs() + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("cannot read %s: %v", l.display(pairsName), redact(err, l.store.home)) + } + defer f.Close() + a := &Agreement{LabHome: l.store.display(l.entry.ID), Types: []TypeAgreement{}} + by := map[string]*TypeAgreement{} + rd := bufio.NewReaderSize(f, maxPairBytes) + for { + raw, rerr := rd.ReadSlice('\n') + if errors.Is(rerr, bufio.ErrBufferFull) { + a.Unreadable++ + for errors.Is(rerr, bufio.ErrBufferFull) { + _, rerr = rd.ReadSlice('\n') + } + if rerr == io.EOF { + break + } + if rerr != nil { + return nil, fmt.Errorf("cannot read %s: %v", l.display(pairsName), redact(rerr, l.store.home)) + } + continue + } + if rerr != nil && rerr != io.EOF { + return nil, fmt.Errorf("cannot read %s: %v", l.display(pairsName), redact(rerr, l.store.home)) + } + a.count(by, bytes.TrimSpace(raw)) + if rerr == io.EOF { + break + } + } + names := make([]string, 0, len(by)) + for n := range by { + names = append(names, n) + } + sort.Strings(names) + compared := 0 + for _, n := range names { + t := by[n] + if t.Compared > 0 { + t.Rate = float64(t.Agreed) / float64(t.Compared) + } + compared += t.Compared + a.Types = append(a.Types, *t) + } + a.Note = fmt.Sprintf("Shadow agreement from the lab home %s: %d pairs (%d compared) over %d judgement type(s), harvested %s.", + a.LabHome, a.Pairs, compared, len(a.Types), stamp()) + return a, nil +} + +// count adds one line of pairs.jsonl to a: a pair to its type's counts and +// bands, a blank line to nothing, and anything else as unreadable. +func (a *Agreement) count(by map[string]*TypeAgreement, line []byte) { + if len(line) == 0 { + return + } + var p Pair + if json.Unmarshal(line, &p) != nil || p.Type == "" || p.Host == "" { + a.Unreadable++ + return + } + a.Pairs++ + t := by[p.Type] + if t == nil { + t = &TypeAgreement{Type: p.Type, Bands: make([]Band, len(bands))} + for i, b := range bands { + t.Bands[i].Band = b.name + } + by[p.Type] = t + } + t.Pairs++ + if p.Skipped != "" || p.Adapter == "" || p.Probability == nil { + t.Skipped++ + return + } + t.Compared++ + agreed := p.Adapter == p.Host + if agreed { + t.Agreed++ + } + for i, b := range bands { + if *p.Probability >= b.lo && *p.Probability < b.hi { + t.Bands[i].Compared++ + if agreed { + t.Bands[i].Agreed++ + } + } + } +} + +// openPairs opens the lab's pairs.jsonl to read: a regular file in the lab +// home, never a link or anything else, and the file the check vetted, not one +// put in its place since; os.ErrNotExist when the lab recorded none. +func (l *lab) openPairs() (*os.File, error) { + fi, err := l.root.Lstat(pairsName) + if err != nil { + return nil, err + } + if !fi.Mode().IsRegular() { + return nil, fsutil.ErrNotRegular + } + f, err := l.root.OpenFile(pairsName, os.O_RDONLY|syscall.O_NONBLOCK, 0) + if err != nil { + return nil, err + } + if st, err := f.Stat(); err != nil || !os.SameFile(fi, st) || !st.Mode().IsRegular() { + f.Close() + return nil, fsutil.ErrNotRegular + } + return f, nil +} + +// agreementDoc renders the harvest's agreement section. +func agreementDoc(a *Agreement) string { + var b strings.Builder + b.WriteString("## Agreement — decision adapter shadow pairs\n\n") + fmt.Fprintf(&b, "From `%s/%s`: %d pairs", a.LabHome, pairsName, a.Pairs) + if a.Unreadable > 0 { + fmt.Fprintf(&b, ", %d line(s) that did not parse and are not counted", a.Unreadable) + } + b.WriteString(". A pair is compared when the adapter answered; one it did not answer (its state\nexceeded the window, or the call failed) is skipped, and the host decided alone.\n\n") + b.WriteString("| Type | Pairs | Compared | Agreed | Skipped | Agreement |\n| --- | --- | --- | --- | --- | --- |\n") + for _, t := range a.Types { + rate := "n/a" + if t.Compared > 0 { + rate = fmt.Sprintf("%.0f%%", t.Rate*100) + } + fmt.Fprintf(&b, "| %s | %d | %d | %d | %d | %s |\n", t.Type, t.Pairs, t.Compared, t.Agreed, t.Skipped, rate) + } + b.WriteString("\nBy the adapter's probability (agreed/compared):\n\n") + for _, t := range a.Types { + parts := make([]string, 0, len(t.Bands)) + for _, bd := range t.Bands { + parts = append(parts, fmt.Sprintf("%s: %d/%d", bd.Band, bd.Agreed, bd.Compared)) + } + fmt.Fprintf(&b, "- %s — %s\n", t.Type, strings.Join(parts, ", ")) + } + fmt.Fprintf(&b, "\nFor the research note that turns a type on:\n\n> %s\n", a.Note) + return b.String() +} diff --git a/internal/core/lab/pairs_test.go b/internal/core/lab/pairs_test.go new file mode 100644 index 0000000000..5487739068 --- /dev/null +++ b/internal/core/lab/pairs_test.go @@ -0,0 +1,208 @@ +package lab + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func prob(p float64) *float64 { return &p } + +// TestRecordPairWritesUnderTheNewestLabHomeAndNothingInTheRepo is the +// decision adapter's first criterion, the lab's half (itd-2609221009495079): +// a shadow pair is appended to pairs.jsonl in the lab home, in the +// machine-scoped store, and nothing is written into the repository. +func TestRecordPairWritesUnderTheNewestLabHomeAndNothingInTheRepo(t *testing.T) { + r, home := fixture(t) + notAbcd(t) + m := mint(t, r) + before := r.Git("status", "--porcelain", "--ignored") + + p := Pair{Type: "duplicate-match", Kind: "choose", StateSHA256: strings.Repeat("a", 64), StateTokens: 12, + Host: "iss-1", Adapter: "iss-1", Probability: prob(0.92), Target: "openrouter/typesafe/jev-1.13"} + id, where, err := RecordPair(r.Root(), p) + if err != nil { + t.Fatalf("RecordPair: %v", err) + } + if id != m.ID || !strings.HasSuffix(where, m.ID+"/pairs.jsonl") || strings.Contains(where, home) { + t.Fatalf("recorded in %s at %s, want lab %s in tilde form", id, where, m.ID) + } + if _, _, err := RecordPair(r.Root(), Pair{Type: "duplicate-match", Kind: "choose", StateSHA256: strings.Repeat("b", 64), + StateTokens: 40000, Host: "none", Target: "openrouter/typesafe/jev-1.13", Skipped: SkippedWindow}); err != nil { + t.Fatal(err) + } + lines := strings.Split(strings.TrimSpace(read(t, filepath.Join(labDir(home, m), "pairs.jsonl"))), "\n") + if len(lines) != 2 { + t.Fatalf("pairs.jsonl holds %d lines, want 2", len(lines)) + } + var got Pair + if err := json.Unmarshal([]byte(lines[0]), &got); err != nil || got.Host != "iss-1" || got.Probability == nil || + *got.Probability != 0.92 || got.At == "" { + t.Fatalf("first pair = %+v, %v", got, err) + } + if fi, err := os.Stat(filepath.Join(labDir(home, m), "pairs.jsonl")); err != nil || fi.Mode().Perm() != 0o600 { + t.Fatalf("pairs.jsonl mode = %v, %v; want 0600", fi, err) + } + if after := r.Git("status", "--porcelain", "--ignored"); after != before { + t.Fatalf("the repository changed:\n%s\nwas\n%s", after, before) + } +} + +func TestRecordPairWithNoLabRefuses(t *testing.T) { + r, _ := fixture(t) + _, _, err := RecordPair(r.Root(), Pair{Type: "duplicate-match", Kind: "choose", StateSHA256: strings.Repeat("a", 64), Host: "x"}) + if !errors.Is(err, ErrNoLab) { + t.Fatalf("err = %v, want ErrNoLab", err) + } +} + +func TestRecordPairRefusesAPairWithoutItsType(t *testing.T) { + r, _ := fixture(t) + notAbcd(t) + mint(t, r) + if _, _, err := RecordPair(r.Root(), Pair{Kind: "choose", StateSHA256: strings.Repeat("a", 64), Host: "x"}); !errors.Is(err, ErrRefused) { + t.Fatalf("err = %v, want a refusal", err) + } +} + +// TestHarvestReportsAgreementPerTypeWithBands is the second criterion: the +// harvest reports agreement per judgement type with counts and probability +// bands, and the note it renders names the lab home and the count. +func TestHarvestReportsAgreementPerTypeWithBands(t *testing.T) { + r, home := fixture(t) + notAbcd(t) + m := mint(t, r) + sha := strings.Repeat("c", 64) + for _, p := range []Pair{ + {Type: "duplicate-match", Host: "a", Adapter: "a", Probability: prob(0.95)}, + {Type: "duplicate-match", Host: "a", Adapter: "b", Probability: prob(0.95)}, + {Type: "duplicate-match", Host: "b", Adapter: "b", Probability: prob(0.6)}, + {Type: "duplicate-match", Host: "b", Skipped: SkippedWindow}, + {Type: "remedy-read", Host: "yes", Adapter: "yes", Probability: prob(0.3)}, + } { + p.Kind, p.StateSHA256, p.Target = "choose", sha, "openrouter/typesafe/jev-1.13" + if _, _, err := RecordPair(r.Root(), p); err != nil { + t.Fatal(err) + } + } + // A line that does not parse is counted, never guessed at. + f, err := os.OpenFile(filepath.Join(labDir(home, m), "pairs.jsonl"), os.O_APPEND|os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + f.WriteString("{not json\n") + f.Close() + + res, err := Harvest(r.Root(), m.ID) + if err != nil { + t.Fatalf("Harvest: %v", err) + } + a := res.Agreement + if a == nil || a.Pairs != 5 || a.Unreadable != 1 || len(a.Types) != 2 { + t.Fatalf("agreement = %+v", a) + } + dm := a.Types[0] + if dm.Type != "duplicate-match" || dm.Pairs != 4 || dm.Compared != 3 || dm.Agreed != 2 || dm.Skipped != 1 { + t.Fatalf("duplicate-match = %+v", dm) + } + bands := map[string]Band{} + for _, b := range dm.Bands { + bands[b.Band] = b + } + if b := bands["0.90-1.00"]; b.Compared != 2 || b.Agreed != 1 { + t.Fatalf("0.90-1.00 band = %+v (bands %+v)", b, dm.Bands) + } + if b := bands["0.50-0.70"]; b.Compared != 1 || b.Agreed != 1 { + t.Fatalf("0.50-0.70 band = %+v", b) + } + if rr := a.Types[1]; rr.Type != "remedy-read" || rr.Compared != 1 || rr.Agreed != 1 { + t.Fatalf("remedy-read = %+v", rr) + } + labHome := StoreDisplay + "/" + m.RootSHA + "/" + m.ID + if a.LabHome != labHome || !strings.Contains(a.Note, labHome) || !strings.Contains(a.Note, "5 pairs") { + t.Fatalf("note = %q, home = %q; want the lab home %s and the count", a.Note, a.LabHome, labHome) + } + doc := read(t, filepath.Join(labDir(home, m), "harvest", "harvest.md")) + for _, want := range []string{"## Agreement", "| duplicate-match | 4 | 3 | 2 | 1 | 67% |", "0.90-1.00: 1/2", a.Note} { + if !strings.Contains(doc, want) { + t.Errorf("harvest.md lacks %q:\n%s", want, doc) + } + } +} + +func TestHarvestWithNoPairsHasNoAgreement(t *testing.T) { + r, home := fixture(t) + notAbcd(t) + m := mint(t, r) + res, err := Harvest(r.Root(), m.ID) + if err != nil { + t.Fatal(err) + } + if res.Agreement != nil { + t.Fatalf("agreement = %+v, want none without pairs", res.Agreement) + } + if strings.Contains(read(t, filepath.Join(labDir(home, m), "harvest", "harvest.md")), "## Agreement") { + t.Fatal("harvest.md carries an agreement section with no pairs recorded") + } +} + +// TestHarvestReadsAPairsFilePastTheDocumentBound: a busy lab's pairs.jsonl +// grows past the bound one document read holds, and the harvest still reads +// every pair, a line at a time; a line longer than a pair may be is counted as +// one that did not parse, and the harvest goes on past it. +func TestHarvestReadsAPairsFilePastTheDocumentBound(t *testing.T) { + r, home := fixture(t) + notAbcd(t) + m := mint(t, r) + line, err := json.Marshal(Pair{At: "2026-10-29T06:00:00Z", Type: "eligibility-residual", Kind: "choose", + StateSHA256: strings.Repeat("d", 64), StateTokens: 300, Host: "no", Adapter: "no", Probability: prob(0.93), + Target: "openrouter/typesafe/jev-1.13", ModelReported: "typesafe/jev-1.13-0901"}) + if err != nil { + t.Fatal(err) + } + line = append(line, '\n') + n := maxDocBytes/len(line) + 1000 + var b strings.Builder + for i := range n { + if i == n/2 { + b.WriteString(strings.Repeat("x", maxPairBytes+10) + "\n") + } + b.Write(line) + } + path := filepath.Join(labDir(home, m), "pairs.jsonl") + if err := os.WriteFile(path, []byte(b.String()), 0o600); err != nil { + t.Fatal(err) + } + if fi, err := os.Stat(path); err != nil || fi.Size() <= maxDocBytes { + t.Fatalf("the fixture is %v bytes, %v; want more than %d", fi, err, maxDocBytes) + } + res, err := Harvest(r.Root(), m.ID) + if err != nil { + t.Fatalf("Harvest over a pairs file past the bound: %v", err) + } + a := res.Agreement + if a == nil || a.Pairs != n || a.Unreadable != 1 || len(a.Types) != 1 || a.Types[0].Agreed != n { + t.Fatalf("agreement = %+v, want %d pairs, all agreed, and one unreadable line", a, n) + } +} + +// TestHarvestRefusesALinkedPairsFile: the harvest reads pairs.jsonl only as a +// regular file in the lab home, never through a link to one elsewhere. +func TestHarvestRefusesALinkedPairsFile(t *testing.T) { + r, home := fixture(t) + notAbcd(t) + m := mint(t, r) + elsewhere := filepath.Join(t.TempDir(), "pairs.jsonl") + if err := os.WriteFile(elsewhere, []byte(`{"type":"remedy-read","kind":"choose","host":"yes"}`+"\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(elsewhere, filepath.Join(labDir(home, m), "pairs.jsonl")); err != nil { + t.Fatal(err) + } + if _, err := Harvest(r.Root(), m.ID); err == nil || !strings.Contains(err.Error(), "pairs.jsonl") { + t.Fatalf("Harvest through a linked pairs file = %v, want a refusal naming it", err) + } +} diff --git a/internal/core/oracle/config.go b/internal/core/oracle/config.go index 28376e8dd4..a00dd92f40 100644 --- a/internal/core/oracle/config.go +++ b/internal/core/oracle/config.go @@ -26,6 +26,12 @@ package oracle // a route the person sets on their own machine may spend their paid key // (the product thinker's ruling AA(b) of 2026-09-29, which reverses the // route half of itd-2609081951381895 Decision 8). +// - A judgement type's route carries a mode (itd-2609221009495079): `host` +// keeps the type on the host and points it nowhere, `shadow:/ +// ` runs the decision adapter beside the host, which still decides, +// and a bare `/` turns the type over to the adapter. A +// shadow route is judged exactly as one turned on: the same skips, the same +// allowlist and the same denylist, before any call. A role takes no mode. // // Every route is checked here, before any call: a repository route to a // provider that holds a key is skipped with a diagnostic naming the machine's @@ -134,11 +140,53 @@ type Target struct { Model string `json:"model"` // Origin names the file the route came from. Origin string `json:"origin"` + // Mode is a judgement type's mode, JudgementShadow or JudgementOn; "" + // on a role's target. + Mode string `json:"mode,omitempty"` } -// String is the route as the configuration spells it. +// The modes a judgement type's route carries (itd-2609221009495079). A role's +// target carries none. +const ( + // JudgementHost keeps the type on the host: the route points nowhere. + JudgementHost = "host" + // JudgementShadow runs the adapter beside the host, which decides. + JudgementShadow = "shadow" + // JudgementOn turns the type over to the adapter. + JudgementOn = "on" +) + +// shadowPrefix spells a shadow route in the configuration. +const shadowPrefix = JudgementShadow + ":" + +// String is the target as /, whatever its mode. func (t Target) String() string { return t.Provider + "/" + t.Model } +// Route is the route as the configuration spells it: a shadow judgement's with +// its shadow: prefix. +func (t Target) Route() string { + if t.Mode == JudgementShadow { + return shadowPrefix + t.String() + } + return t.String() +} + +// routeValue splits a route's mode from its / text. Only a +// judgement type's route has a mode: host (target ""), shadow (the prefix +// cut) or on; a role's text is returned whole, with no mode. +func routeValue(key, text string) (target, mode string) { + if key != judgementsKey { + return text, "" + } + if text == JudgementHost { + return "", JudgementHost + } + if t, ok := strings.CutPrefix(text, shadowPrefix); ok { + return t, JudgementShadow + } + return text, JudgementOn +} + // DenyEntry is one denylist entry and the layer that added it. type DenyEntry struct { Pattern string `json:"pattern"` @@ -447,9 +495,12 @@ func (c *APIConfig) readRoutes(s *layered.Stack, key string, into map[string]Tar // The order matters: skipKeyedRoute is judged before skipUnlistedRoute, // so a keyed provider's list is never consulted on a repository's // behalf, and its route is skipped as one that would spend a key - // whatever model it names. + // whatever model it names. A judgement type's route that turns the + // type on is the person's own to set (skipTurnedOnRoute), so one + // from any other layer is skipped last, after the reasons above. for len(found) > 0 && (c.skipMalformedRoute(key, name, found[0]) || c.skipKeyedRoute(key, name, found[0]) || - c.skipUnconfiguredRoute(key, name, found) || c.skipUnlistedRoute(key, name, found[0])) { + c.skipUnconfiguredRoute(key, name, found) || c.skipUnlistedRoute(key, name, found[0]) || + c.skipTurnedOnRoute(key, name, found[0])) { found = found[1:] } if len(found) == 0 { @@ -466,9 +517,20 @@ func (c *APIConfig) readRoutes(s *layered.Stack, key string, into map[string]Tar if err != nil { return fmt.Errorf("oracle adapter: %s: %w", where, err) } + spelled := text + text, mode := routeValue(key, text) + if mode == JudgementHost { + // The winning layer keeps the type on the host: no route, and + // the lower layers' routes to it do not apply. + continue + } provider, model, ok := strings.Cut(text, "/") if !ok || provider == "" || model == "" { - return fmt.Errorf("oracle adapter: %s is %q; a route is /", where, layered.BoundKey(text)) + forms := "a route is /" + if key == judgementsKey { + forms = "a judgement type's route is host, shadow:/ or /" + } + return fmt.Errorf("oracle adapter: %s is %q; %s", where, layered.BoundKey(spelled), forms) } if _, configured := c.providers[provider]; !configured { c.Diagnostics = append(c.Diagnostics, fmt.Sprintf("oracle adapter: %s points at provider %q, which is not configured on this machine; "+ @@ -478,7 +540,7 @@ func (c *APIConfig) readRoutes(s *layered.Stack, key string, into map[string]Tar if err := c.Admit(provider, model); err != nil { return fmt.Errorf("oracle adapter: %s points at %s, %w", where, layered.BoundKey(text), err) } - into[name] = Target{Provider: provider, Model: model, Origin: win.Origin} + into[name] = Target{Provider: provider, Model: model, Origin: win.Origin, Mode: mode} } return nil } @@ -507,7 +569,11 @@ func (c *APIConfig) skipMalformedRoute(key, name string, fd layered.Found) bool } what := "is not a string" if text, err := layered.Decode[string](fd.Raw); err == nil { - provider, model, ok := strings.Cut(text, "/") + target, mode := routeValue(key, text) + if mode == JudgementHost { + return false + } + provider, model, ok := strings.Cut(target, "/") if ok && provider != "" && model != "" { return false } @@ -536,6 +602,7 @@ func (c *APIConfig) skipKeyedRoute(key, name string, fd layered.Found) bool { if err != nil { return false } + text, _ = routeValue(key, text) provider, model, ok := strings.Cut(text, "/") if !ok || provider == "" || model == "" { return false @@ -581,6 +648,7 @@ func (c *APIConfig) skipUnconfiguredRoute(key, name string, found []layered.Foun if err != nil { return false } + text, _ = routeValue(key, text) provider, _, ok := strings.Cut(text, "/") if !ok || provider == "" { return false @@ -596,6 +664,43 @@ func (c *APIConfig) skipUnconfiguredRoute(key, name string, found []layered.Foun return true } +// skipTurnedOnRoute reports whether one layer's route to name is a judgement +// type's route that turns the type on, from a layer that is not the person's +// own, and says so in a diagnostic when it is. A type turned on hands its +// decision to the adapter, and only the person's own configuration may do +// that: the machine's file under their home, or their own flag for one run, +// the layers runner.personal names for a role handed to a runner (rulings RN2 +// and OC2 of 2026-10-02). The rule is restated here rather than shared, +// because runner imports this package. Skipping the route lets the next +// layer's route to the type apply, the machine's in its own mode, or the +// host when the machine sets none, so a repository never turns a machine's +// shadow into on. A repository's host and shadow are not skipped here: host +// keeps work on the host, and in shadow the host still decides. A route the +// denylist matches is left to the refusal below, which no layer softens. +func (c *APIConfig) skipTurnedOnRoute(key, name string, fd layered.Found) bool { + if key != judgementsKey || fd.Layer == layered.Machine || fd.Layer == layered.Flag { + return false + } + text, err := layered.Decode[string](fd.Raw) + if err != nil { + return false + } + target, mode := routeValue(key, text) + if mode != JudgementOn { + return false + } + _, model, _ := strings.Cut(target, "/") + if _, denied := Denied(c.denylist, model); denied { + return false + } + c.Diagnostics = append(c.Diagnostics, fmt.Sprintf("oracle adapter: %s (%s layer): %s.%s is %s, which turns the type on; "+ + "only the person's own configuration (%s) turns a judgement type on, so this route is skipped and the route %s sets "+ + "for the type, or the host, applies in its place: set it in %s, or spell it %s to run the adapter beside the host", + fd.Origin, fd.Layer, key, name, quoteUntrusted(text), layered.Config.MachineOrigin(), layered.Config.MachineOrigin(), + layered.Config.MachineOrigin(), quoteUntrusted(shadowPrefix+target))) + return true +} + // skipUnlistedRoute reports whether one layer's route to name is a route from // any layer but the machine's to a configured provider that holds no key, // naming a model that provider does not list, and says so in a diagnostic @@ -616,6 +721,7 @@ func (c *APIConfig) skipUnlistedRoute(key, name string, fd layered.Found) bool { if err != nil { return false } + text, _ = routeValue(key, text) provider, model, ok := strings.Cut(text, "/") if !ok || provider == "" || model == "" { return false diff --git a/internal/core/oracle/config_test.go b/internal/core/oracle/config_test.go index 3f462e913b..7f75243c1f 100644 --- a/internal/core/oracle/config_test.go +++ b/internal/core/oracle/config_test.go @@ -164,7 +164,8 @@ func TestARepositoryRouteToAnUnlistedModelIsSkipped(t *testing.T) { f.machineConfig(`{"oracle":{"api":{` + localBlock + `}` + tc.machine + `}}`) other := `"roles":{"scribe":"local/qwen/qwen3-8b"}` if tc.family == rolesKey { - other = `"judgements":{"duplicate-match":"local/qwen/qwen3-8b"}` + // In shadow: only the person's own file turns a type on. + other = `"judgements":{"duplicate-match":"shadow:local/qwen/qwen3-8b"}` } f.repoConfig(`{"oracle":{` + tc.repo + `,` + other + `}}`) c, err := LoadAPI(f.roots) @@ -280,7 +281,8 @@ func TestARepositoryRouteToAKeyedProviderIsSkipped(t *testing.T) { // A keyless route beside the refused one: the skip costs nothing else. other := `"roles":{"scribe":"local/qwen/qwen3-8b"}` if tc.family == rolesKey { - other = `"judgements":{"duplicate-match":"local/qwen/qwen3-8b"}` + // In shadow: only the person's own file turns a type on. + other = `"judgements":{"duplicate-match":"shadow:local/qwen/qwen3-8b"}` } f.repoConfig(`{"oracle":{` + tc.repo + `,` + other + `}}`) c, err := LoadAPI(f.roots) @@ -479,16 +481,18 @@ func TestAMachineRouteToAKeyedProviderIsAdmitted(t *testing.T) { // TestARepositoryRouteToAKeylessProviderIsAdmitted: a provider whose block // names no key spends no key, so the repository may point at it, and its route -// wins over the machine's as every repository route does. +// wins over the machine's as every repository route does; a judgement type's +// route in shadow, since only the person's own file turns a type on +// (TestOnlyThePersonsOwnConfigurationTurnsAJudgementTypeOn). func TestARepositoryRouteToAKeylessProviderIsAdmitted(t *testing.T) { f := newFx(t) f.machineConfig(`{"oracle":{"api":{` + openrouterBlock + `,` + localBlock + `},"roles":{"scribe":"openrouter/typesafe/jev-1.13"}}}`) - f.repoConfig(`{"oracle":{"roles":{"scribe":"local/qwen/qwen3-8b"},"judgements":{"duplicate-match":"local/qwen/qwen3-8b"}}}`) + f.repoConfig(`{"oracle":{"roles":{"scribe":"local/qwen/qwen3-8b"},"judgements":{"duplicate-match":"shadow:local/qwen/qwen3-8b"}}}`) c := f.loadAPI() if tgt, ok := c.Role("scribe"); !ok || tgt.Provider != "local" || tgt.Origin != ".abcd/config.json" { t.Fatalf("role = %+v, %v", tgt, ok) } - if tgt, ok := c.Judgement("duplicate-match"); !ok || tgt.Provider != "local" { + if tgt, ok := c.Judgement("duplicate-match"); !ok || tgt.Provider != "local" || tgt.Mode != JudgementShadow { t.Fatalf("judgement = %+v, %v", tgt, ok) } } diff --git a/internal/core/oracle/judgement_mode_test.go b/internal/core/oracle/judgement_mode_test.go new file mode 100644 index 0000000000..056ab0530d --- /dev/null +++ b/internal/core/oracle/judgement_mode_test.go @@ -0,0 +1,179 @@ +package oracle + +import ( + "strings" + "testing" + + "github.com/intentdriven/abcd/internal/abcdhome" +) + +// A judgement type's route carries a mode (itd-2609221009495079, +// spc-2609221011151661 scope 3): `host` keeps the type on the host, +// `shadow:/` runs the adapter beside the host, which still +// decides, and a bare `/` turns the type over to the adapter. +// The allowlist is judged for a shadow route exactly as for one turned on, when +// the configuration is read and before any call (criterion 4). + +func TestAJudgementRouteCarriesItsMode(t *testing.T) { + f := newFx(t) + f.machineConfig(`{"oracle":{"api":{` + openrouterBlock + `}, + "judgements":{"duplicate-match":"shadow:openrouter/typesafe/jev-1.13", + "remedy-read":"openrouter/typesafe/jev-latest", + "memory-check":"host"}}}`) + c := f.loadAPI() + if len(c.Diagnostics) != 0 { + t.Fatalf("diagnostics = %q, want none", c.Diagnostics) + } + tgt, ok := c.Judgement("duplicate-match") + if !ok || tgt.Provider != "openrouter" || tgt.Model != "typesafe/jev-1.13" || tgt.Mode != JudgementShadow { + t.Fatalf("shadow judgement = %+v, %v", tgt, ok) + } + if got := tgt.Route(); got != "shadow:openrouter/typesafe/jev-1.13" { + t.Fatalf("Route() = %q", got) + } + tgt, ok = c.Judgement("remedy-read") + if !ok || tgt.Model != "typesafe/jev-latest" || tgt.Mode != JudgementOn { + t.Fatalf("judgement turned on = %+v, %v", tgt, ok) + } + if got := tgt.Route(); got != "openrouter/typesafe/jev-latest" { + t.Fatalf("Route() = %q", got) + } + if tgt, ok := c.Judgement("memory-check"); ok { + t.Fatalf("a host judgement is pointed at %+v; host means no route", tgt) + } + // A role carries no mode: the shadow prefix is the judgements' alone. + for _, r := range c.Routes() { + if r.Kind == "role" && r.Target.Mode != "" { + t.Fatalf("role %s carries mode %q", r.Name, r.Target.Mode) + } + } +} + +func TestAShadowRouteToAnUnlistedModelIsRefusedBeforeAnyCall(t *testing.T) { + f := newFx(t) + f.machineConfig(`{"oracle":{"api":{` + openrouterBlock + `}, + "judgements":{"duplicate-match":"shadow:openrouter/mistral/small"}}}`) + err := f.loadAPIErr() + for _, want := range []string{"not on openrouter's list", "typesafe/jev-1.13, typesafe/jev-latest", "refused before any call"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("refusal %q does not name %q", err, want) + } + } +} + +func TestADenylistedShadowRouteIsRefused(t *testing.T) { + f := newFx(t) + f.machineConfig(`{"oracle":{"api":{` + openrouterBlock + `},"denylist":["typesafe/*"], + "judgements":{"duplicate-match":"shadow:openrouter/typesafe/jev-1.13"}}}`) + if err := f.loadAPIErr(); !strings.Contains(err.Error(), "typesafe/*") { + t.Fatalf("refusal = %v, want the denylist entry named", err) + } +} + +func TestARepositoryShadowRouteToAKeyedProviderIsSkipped(t *testing.T) { + f := newFx(t) + f.machineConfig(`{"oracle":{"api":{` + openrouterBlock + `}}}`) + f.repoConfig(`{"oracle":{"judgements":{"duplicate-match":"shadow:openrouter/typesafe/jev-1.13"}}}`) + c := f.loadAPI() + if tgt, ok := c.Judgement("duplicate-match"); ok { + t.Fatalf("a repository's shadow route spends the person's key: %+v", tgt) + } + if len(c.Diagnostics) != 1 || !strings.Contains(c.Diagnostics[0], "holds a key") { + t.Fatalf("diagnostics = %q, want the keyed-route skip", c.Diagnostics) + } +} + +func TestARepositoryHostJudgementOverridesTheMachineRoute(t *testing.T) { + f := newFx(t) + f.machineConfig(`{"oracle":{"api":{` + localBlock + `},"judgements":{"duplicate-match":"local/qwen/qwen3-8b"}}}`) + f.repoConfig(`{"oracle":{"judgements":{"duplicate-match":"host"}}}`) + c := f.loadAPI() + if tgt, ok := c.Judgement("duplicate-match"); ok { + t.Fatalf("judgement = %+v; the repository's host keeps it on the host", tgt) + } + if len(c.Diagnostics) != 0 { + t.Fatalf("diagnostics = %q, want none: host is not a malformed route", c.Diagnostics) + } +} + +func TestAMalformedShadowRouteIsRefusedOnTheMachine(t *testing.T) { + f := newFx(t) + f.machineConfig(`{"oracle":{"api":{` + openrouterBlock + `},"judgements":{"duplicate-match":"shadow:openrouter"}}}`) + // The refusal quotes the value as the file spells it, and names every + // form a judgement type's route takes. + err := f.loadAPIErr() + for _, want := range []string{`"shadow:openrouter"`, "host, shadow:/ or /"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("refusal %q does not name %q", err, want) + } + } + // A role takes no mode: shadow: on a role names a provider called + // "shadow:openrouter", which is not a plain name and not configured. + f = newFx(t) + f.machineConfig(`{"oracle":{"api":{` + openrouterBlock + `},"roles":{"scribe":"shadow:openrouter/typesafe/jev-1.13"}}}`) + c := f.loadAPI() + if tgt, ok := c.Role("scribe"); ok { + t.Fatalf("a role took the shadow mode: %+v", tgt) + } +} + +// A judgement type is turned over to an adapter only from the person's own +// configuration: the machine's file, or their own flag for one run, the same +// rule runner.personal applies to a role handed to a runner. A route the +// repository's file turns on is skipped with one diagnostic, and the +// machine's route to the type applies in its place, its mode with it, or the +// host when the machine sets none. The repository may still keep a type on +// the host, or run the adapter beside the host in shadow. +func TestOnlyThePersonsOwnConfigurationTurnsAJudgementTypeOn(t *testing.T) { + const repoOn = `{"oracle":{"judgements":{"eligibility-residual":"local/qwen/qwen3-8b"}}}` + for name, tc := range map[string]struct { + machine string + wantOK bool + wantMode string + }{ + "no machine route, so the host": {machine: ``}, + "the machine's shadow stays shadow": {machine: `,"judgements":{"eligibility-residual":"shadow:local/qwen/qwen3-8b"}`, + wantOK: true, wantMode: JudgementShadow}, + "the machine's own on applies": {machine: `,"judgements":{"eligibility-residual":"local/qwen/qwen3-8b"}`, + wantOK: true, wantMode: JudgementOn}, + "the machine's host stays host": {machine: `,"judgements":{"eligibility-residual":"host"}`}, + } { + t.Run(name, func(t *testing.T) { + f := newFx(t) + f.machineConfig(`{"oracle":{"api":{` + localBlock + `}` + tc.machine + `}}`) + f.repoConfig(repoOn) + c := f.loadAPI() + tgt, ok := c.Judgement("eligibility-residual") + if ok != tc.wantOK { + t.Fatalf("judgement = %+v, %v; want routed %v", tgt, ok, tc.wantOK) + } + if ok && (tgt.Mode != tc.wantMode || tgt.Origin != abcdhome.Display("config.json")) { + t.Fatalf("judgement = %+v; want the machine's route in mode %q", tgt, tc.wantMode) + } + if len(c.Diagnostics) != 1 { + t.Fatalf("diagnostics = %q, want exactly one naming the skipped route", c.Diagnostics) + } + for _, want := range []string{".abcd/config.json (repo layer)", "oracle.judgements.eligibility-residual", + "person's own configuration", "skipped", "shadow:local/qwen/qwen3-8b"} { + if !strings.Contains(c.Diagnostics[0], want) { + t.Errorf("diagnostic %q does not name %q", c.Diagnostics[0], want) + } + } + }) + } +} + +// The repository's own shadow and host stand: neither hands the decision over. +func TestARepositoryMayKeepAJudgementTypeInShadow(t *testing.T) { + f := newFx(t) + f.machineConfig(`{"oracle":{"api":{` + localBlock + `}}}`) + f.repoConfig(`{"oracle":{"judgements":{"eligibility-residual":"shadow:local/qwen/qwen3-8b"}}}`) + c := f.loadAPI() + tgt, ok := c.Judgement("eligibility-residual") + if !ok || tgt.Mode != JudgementShadow || tgt.Origin != ".abcd/config.json" { + t.Fatalf("judgement = %+v, %v; want the repository's shadow route", tgt, ok) + } + if len(c.Diagnostics) != 0 { + t.Fatalf("diagnostics = %q, want none", c.Diagnostics) + } +} diff --git a/internal/surface/cli/ahoy_connect.go b/internal/surface/cli/ahoy_connect.go index 6f2e2c1237..57cdeccc2d 100644 --- a/internal/surface/cli/ahoy_connect.go +++ b/internal/surface/cli/ahoy_connect.go @@ -134,8 +134,10 @@ func runAhoyProviders(cmd *cobra.Command, cwd string, asJSON bool) error { } else { line("denylist (oracle.denylist), which refuses a model even when a provider lists it: " + strings.Join(deny, ", ")) } + // Route spells a judgement route as the configuration does, so a + // shadow route never reads as the bare form that turns a type on. for _, r := range b.Routes { - line(fmt.Sprintf("%s %s -> %s (%s)", r.Kind, r.Name, r.Target, r.Target.Origin)) + line(fmt.Sprintf("%s %s -> %s (%s)", r.Kind, r.Name, r.Target.Route(), r.Target.Origin)) } for _, d := range b.Diagnostics { line(d) @@ -263,13 +265,18 @@ func newAhoyConnectCommand(asJSON *bool) *cobra.Command { } // A route to a provider that holds a key is the machine's alone // (the ruling AA(b) of 2026-09-29, enforced by oracle.LoadAPI), so - // the repository's file is offered only for a keyless one. - where := layered.Config.MachineOrigin() + // the repository's file is offered only for a keyless one. Even + // then a judgement type is turned on only from the person's own + // configuration, so both files are offered its shadow spelling + // and the bare one is named for the machine's file alone. + machine, route := layered.Config.MachineOrigin(), res.Provider+"/"+res.Models[0] if res.KeyHome == oracle.KeyHomeNone { - where = layered.Config.RepoOrigin() + " or " + where + line(fmt.Sprintf("point a role at it with oracle.roles. = %q in %s or %s", route, layered.Config.RepoOrigin(), machine)) + line(fmt.Sprintf("put a judgement type in shadow on it with oracle.judgements. = %q in either file; the bare %q turns the type on only from %s", + "shadow:"+route, route, machine)) + } else { + line(fmt.Sprintf("point a role or a judgement type at it with oracle.roles. or oracle.judgements. = %q in %s", route, machine)) } - line(fmt.Sprintf("point a role or a judgement type at it with oracle.roles. or oracle.judgements. = %q in %s", - res.Provider+"/"+res.Models[0], where)) line(dispatchNote + ".") }) }, diff --git a/internal/surface/cli/ahoy_connect_test.go b/internal/surface/cli/ahoy_connect_test.go index d86ba50fc6..d3e4acca1c 100644 --- a/internal/surface/cli/ahoy_connect_test.go +++ b/internal/surface/cli/ahoy_connect_test.go @@ -148,8 +148,10 @@ func TestAhoyConnectVerifiesThenWrites(t *testing.T) { } // TestAhoyConnectKeylessOffersBothFiles: a provider that takes no key spends -// no key, so a route to it may sit in the repository's file or the machine's, -// and the advice names both. +// no key, so a role's route to it may sit in the repository's file or the +// machine's, and the advice names both. A judgement type is offered the shadow +// spelling, which both files honour, and told that the bare spelling turns the +// type on only from the machine's file, the person's own. func TestAhoyConnectKeylessOffersBothFiles(t *testing.T) { hermeticEnv(t) t.Chdir(t.TempDir()) @@ -158,8 +160,20 @@ func TestAhoyConnectKeylessOffersBothFiles(t *testing.T) { if err != nil { t.Fatalf("ahoy connect: %v\n%s", err, out) } - if calls.Load() != 1 || !strings.Contains(string(out), `= "local/qwen/qwen3-8b" in .abcd/config.json or `+abcdhome.Display("config.json")) { - t.Fatalf("ahoy connect (%d call(s)):\n%s", calls.Load(), out) + if calls.Load() != 1 { + t.Fatalf("ahoy connect made %d call(s):\n%s", calls.Load(), out) + } + for _, want := range []string{ + `oracle.roles. = "local/qwen/qwen3-8b" in .abcd/config.json or ` + abcdhome.Display("config.json"), + `oracle.judgements. = "shadow:local/qwen/qwen3-8b" in either file`, + `turns the type on only from ` + abcdhome.Display("config.json"), + } { + if !strings.Contains(string(out), want) { + t.Errorf("ahoy connect does not say %q:\n%s", want, out) + } + } + if strings.Contains(string(out), `oracle.judgements. = "local/qwen/qwen3-8b" in .abcd/config.json`) { + t.Errorf("ahoy connect offers the repository's file a judgement route that turns the type on:\n%s", out) } } @@ -386,3 +400,28 @@ func TestAhoyConnectTakesStreamUsage(t *testing.T) { t.Fatalf("config.json:\n%s", raw) } } + +// TestAhoyProvidersSpellsAShadowJudgementRoute: the text board spells a +// judgement route as the configuration does, so a shadow route reads as +// shadow and never as the bare / that turns a type on. +func TestAhoyProvidersSpellsAShadowJudgementRoute(t *testing.T) { + hermeticEnv(t) + t.Chdir(t.TempDir()) + cfg := abcdhome.Path(os.Getenv("HOME"), "config.json") + if err := os.MkdirAll(filepath.Dir(cfg), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(cfg, []byte(`{"oracle":{"api":{"local":{"base_url":"http://localhost:11434/v1","models":["m"]}},`+ + `"judgements":{"remedy-read":"shadow:local/m","duplicate-match":"local/m"}}}`), 0o600); err != nil { + t.Fatal(err) + } + out, err := runCLIErr(t, "ahoy", "--providers") + if err != nil { + t.Fatalf("ahoy --providers: %v\n%s", err, out) + } + for _, want := range []string{"judgement remedy-read -> shadow:local/m (", "judgement duplicate-match -> local/m ("} { + if !strings.Contains(string(out), want) { + t.Errorf("the board does not say %q:\n%s", want, out) + } + } +} diff --git a/internal/surface/cli/lab.go b/internal/surface/cli/lab.go index a0c9e59710..a7776b25a0 100644 --- a/internal/surface/cli/lab.go +++ b/internal/surface/cli/lab.go @@ -357,4 +357,22 @@ func renderHarvest(w io.Writer, res lab.Harvested) { fmt.Fprintf(w, " %s %s\n", c.Finding, termsafe.Sanitize(c.Command)) } } + // The decision adapter's shadow agreement (itd-2609221009495079): one + // line per judgement type, then the research note's line. + if a := res.Agreement; a != nil { + for _, t := range a.Types { + // A type with nothing compared has no rate: n/a, as harvest.md + // prints it, never 0%. + rate := "n/a" + if t.Compared > 0 { + rate = fmt.Sprintf("%.0f%%", t.Rate*100) + } + fmt.Fprintf(w, "agreement %s: %d/%d compared agree (%s), %d skipped\n", + termsafe.Sanitize(t.Type), t.Agreed, t.Compared, rate, t.Skipped) + } + if a.Unreadable > 0 { + fmt.Fprintf(w, " %s in pairs.jsonl did not parse and are not counted\n", countOf(a.Unreadable, "line")) + } + fmt.Fprintf(w, "note: %s\n", termsafe.Sanitize(a.Note)) + } } diff --git a/internal/surface/cli/lab_surface_test.go b/internal/surface/cli/lab_surface_test.go index a86fd05357..2f46323652 100644 --- a/internal/surface/cli/lab_surface_test.go +++ b/internal/surface/cli/lab_surface_test.go @@ -148,3 +148,70 @@ func TestLabVerbRefusesARequestItCannotServe(t *testing.T) { t.Errorf("lab outside a checkout: exit %d, %q", code, errOut) } } + +// The harvest's agreement report reaches the CLI (itd-2609221009495079 +// criterion 2): a lab holding decision-adapter shadow pairs prints the +// agreement per type and the research note's line, naming the lab home and the +// count, in the text and in --json, with no home path in either. +func TestLabHarvestPrintsTheShadowAgreement(t *testing.T) { + home, repo := labCheckout(t) + code, out, errOut := runLab(t, "lab", "--json", "mint", "does", "the", "adapter", "agree?") + if code != 0 { + t.Fatalf("lab mint exit %d: %s%s", code, out, errOut) + } + var m struct { + ID string `json:"id"` + } + if err := json.Unmarshal([]byte(out), &m); err != nil { + t.Fatal(err) + } + p := 0.9 + for _, adapter := range []string{"no", "yes"} { + if _, _, err := lab.RecordPair(repo, lab.Pair{Type: "eligibility-residual", Kind: "choose", StateSHA256: strings.Repeat("d", 64), + Host: "no", Adapter: adapter, Probability: &p, Target: "openrouter/typesafe/jev-1.13"}); err != nil { + t.Fatal(err) + } + } + code, out, _ = runLab(t, "lab", "harvest", m.ID) + if code != 0 || !strings.Contains(out, "agreement eligibility-residual: 1/2 compared agree (50%), 0 skipped") || + !strings.Contains(out, "2 pairs (2 compared)") || !strings.Contains(out, m.ID) { + t.Fatalf("harvest (exit %d):\n%s", code, out) + } + code, out, _ = runLab(t, "lab", "--json", "harvest", m.ID) + var res struct { + Agreement struct { + Pairs int `json:"pairs"` + Note string `json:"note"` + } `json:"agreement"` + } + if code != 0 || json.Unmarshal([]byte(out), &res) != nil || res.Agreement.Pairs != 2 || !strings.Contains(res.Agreement.Note, m.ID) { + t.Fatalf("harvest --json (exit %d):\n%s", code, out) + } + if strings.Contains(out, home) { + t.Fatalf("harvest --json leaked the home path:\n%s", out) + } +} + +// A type whose pairs were all skipped has nothing compared, so the harvest +// prints its agreement as n/a, as harvest.md does, never as 0%. +func TestLabHarvestPrintsNAForATypeWithNothingCompared(t *testing.T) { + _, repo := labCheckout(t) + code, out, errOut := runLab(t, "lab", "--json", "mint", "does", "the", "adapter", "agree?") + if code != 0 { + t.Fatalf("lab mint exit %d: %s%s", code, out, errOut) + } + var m struct { + ID string `json:"id"` + } + if err := json.Unmarshal([]byte(out), &m); err != nil { + t.Fatal(err) + } + if _, _, err := lab.RecordPair(repo, lab.Pair{Type: "remedy-read", Kind: "choose", StateSHA256: strings.Repeat("e", 64), + Host: "no", Target: "openrouter/typesafe/jev-1.13", Skipped: lab.SkippedWindow}); err != nil { + t.Fatal(err) + } + code, out, _ = runLab(t, "lab", "harvest", m.ID) + if code != 0 || !strings.Contains(out, "agreement remedy-read: 0/0 compared agree (n/a), 1 skipped") { + t.Fatalf("harvest (exit %d):\n%s", code, out) + } +}