From 1fd70c6055e07bf4530f552a11e79a867b3beaf6 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 29 Oct 2026 04:57:40 +0000 Subject: [PATCH 01/20] docs: name the decision adapter spec's footprint as real paths The packages line carried prose (the OpenAI-compatible client adapter, callers across capture, intent, memory and lint), which a reservation would take literally. It now names the paths the build changes. Refs: iss-2610241549334990 Assisted-by: Claude:claude-opus-5-5 --- ...-2609221011151661-abcd-s-typed-judgements-can-run-through.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.abcd/development/specs/open/spc-2609221011151661-abcd-s-typed-judgements-can-run-through.md b/.abcd/development/specs/open/spc-2609221011151661-abcd-s-typed-judgements-can-run-through.md index e2867c2679..bc0f9912de 100644 --- a/.abcd/development/specs/open/spc-2609221011151661-abcd-s-typed-judgements-can-run-through.md +++ b/.abcd/development/specs/open/spc-2609221011151661-abcd-s-typed-judgements-can-run-through.md @@ -30,7 +30,7 @@ One interface, one router with three modes, one adapter over the API adapter's c ## Footprint -- packages: internal/core/decide, internal/core/lab, the OpenAI-compatible client adapter, and callers across capture, intent, memory and lint +- packages: internal/core/decide, internal/core/lab, internal/adapter/openaiapi, internal/core/oracle/config.go, internal/core/oracle/judgement_mode_test.go, internal/core/oracle/connect_guide.go, internal/core/implement/loop/drain_decide.go, internal/core/implement/loop/drain_decide_test.go, internal/surface/cli/lab.go, internal/surface/cli/lab_surface_test.go, internal/README.md, commands/lab.md, .abcd/development/brief/04-surfaces/31-lab.md, .abcd/development/brief/05-internals/02-adapters.md, .abcd/development/brief/05-internals/03-configuration.md, .abcd/development/intents/planned/itd-2609221009495079-abcd-s-typed-judgements-can-run-through.md, .abcd/work/issues/open/iss-2610262033055327-the-decision-adapter-intent-treats.md, .abcd/work/issues/resolved/iss-2610262033055327-the-decision-adapter-intent-treats.md, .abcd/work/issues/open/iss-2610262037348735-the-decision-adapter-intent-needs-lab.md, .abcd/work/issues/resolved/iss-2610262037348735-the-decision-adapter-intent-needs-lab.md - tests: the router's three modes with a fake adapter; the pairs file; the harvest's rates; the window skip; the fallback receipt ## How the criteria are satisfied From fc680143d7f353d5f7c3b8e9144eff8ab73d30bf Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:01:10 +0000 Subject: [PATCH 02/20] feat: read a judgement type's route as host, shadow or on oracle.judgements. now takes host, shadow:/ or /. A shadow route passes the same skips, allowlist and denylist as one turned on, when the configuration is read and before any call; host keeps the type on the host and points it nowhere. Assisted-by: Claude:claude-opus-5-5 --- internal/core/oracle/config.go | 67 +++++++++++- internal/core/oracle/judgement_mode_test.go | 111 ++++++++++++++++++++ 2 files changed, 175 insertions(+), 3 deletions(-) create mode 100644 internal/core/oracle/judgement_mode_test.go diff --git a/internal/core/oracle/config.go b/internal/core/oracle/config.go index 28376e8dd4..79ca7d912e 100644 --- a/internal/core/oracle/config.go +++ b/internal/core/oracle/config.go @@ -26,6 +26,12 @@ package oracle // a route the person sets on their own machine may spend their paid key // (the product thinker's ruling AA(b) of 2026-09-29, which reverses the // route half of itd-2609081951381895 Decision 8). +// - A judgement type's route carries a mode (itd-2609221009495079): `host` +// keeps the type on the host and points it nowhere, `shadow:/ +// ` runs the decision adapter beside the host, which still decides, +// and a bare `/` turns the type over to the adapter. A +// shadow route is judged exactly as one turned on: the same skips, the same +// allowlist and the same denylist, before any call. A role takes no mode. // // Every route is checked here, before any call: a repository route to a // provider that holds a key is skipped with a diagnostic naming the machine's @@ -134,11 +140,53 @@ type Target struct { Model string `json:"model"` // Origin names the file the route came from. Origin string `json:"origin"` + // Mode is a judgement type's mode, JudgementShadow or JudgementOn; "" + // on a role's target. + Mode string `json:"mode,omitempty"` } -// String is the route as the configuration spells it. +// The modes a judgement type's route carries (itd-2609221009495079). A role's +// target carries none. +const ( + // JudgementHost keeps the type on the host: the route points nowhere. + JudgementHost = "host" + // JudgementShadow runs the adapter beside the host, which decides. + JudgementShadow = "shadow" + // JudgementOn turns the type over to the adapter. + JudgementOn = "on" +) + +// shadowPrefix spells a shadow route in the configuration. +const shadowPrefix = JudgementShadow + ":" + +// String is the target as /, whatever its mode. func (t Target) String() string { return t.Provider + "/" + t.Model } +// Route is the route as the configuration spells it: a shadow judgement's with +// its shadow: prefix. +func (t Target) Route() string { + if t.Mode == JudgementShadow { + return shadowPrefix + t.String() + } + return t.String() +} + +// routeValue splits a route's mode from its / text. Only a +// judgement type's route has a mode: host (target ""), shadow (the prefix +// cut) or on; a role's text is returned whole, with no mode. +func routeValue(key, text string) (target, mode string) { + if key != judgementsKey { + return text, "" + } + if text == JudgementHost { + return "", JudgementHost + } + if t, ok := strings.CutPrefix(text, shadowPrefix); ok { + return t, JudgementShadow + } + return text, JudgementOn +} + // DenyEntry is one denylist entry and the layer that added it. type DenyEntry struct { Pattern string `json:"pattern"` @@ -466,6 +514,12 @@ func (c *APIConfig) readRoutes(s *layered.Stack, key string, into map[string]Tar if err != nil { return fmt.Errorf("oracle adapter: %s: %w", where, err) } + text, mode := routeValue(key, text) + if mode == JudgementHost { + // The winning layer keeps the type on the host: no route, and + // the lower layers' routes to it do not apply. + continue + } provider, model, ok := strings.Cut(text, "/") if !ok || provider == "" || model == "" { return fmt.Errorf("oracle adapter: %s is %q; a route is /", where, layered.BoundKey(text)) @@ -478,7 +532,7 @@ func (c *APIConfig) readRoutes(s *layered.Stack, key string, into map[string]Tar if err := c.Admit(provider, model); err != nil { return fmt.Errorf("oracle adapter: %s points at %s, %w", where, layered.BoundKey(text), err) } - into[name] = Target{Provider: provider, Model: model, Origin: win.Origin} + into[name] = Target{Provider: provider, Model: model, Origin: win.Origin, Mode: mode} } return nil } @@ -507,7 +561,11 @@ func (c *APIConfig) skipMalformedRoute(key, name string, fd layered.Found) bool } what := "is not a string" if text, err := layered.Decode[string](fd.Raw); err == nil { - provider, model, ok := strings.Cut(text, "/") + target, mode := routeValue(key, text) + if mode == JudgementHost { + return false + } + provider, model, ok := strings.Cut(target, "/") if ok && provider != "" && model != "" { return false } @@ -536,6 +594,7 @@ func (c *APIConfig) skipKeyedRoute(key, name string, fd layered.Found) bool { if err != nil { return false } + text, _ = routeValue(key, text) provider, model, ok := strings.Cut(text, "/") if !ok || provider == "" || model == "" { return false @@ -581,6 +640,7 @@ func (c *APIConfig) skipUnconfiguredRoute(key, name string, found []layered.Foun if err != nil { return false } + text, _ = routeValue(key, text) provider, _, ok := strings.Cut(text, "/") if !ok || provider == "" { return false @@ -616,6 +676,7 @@ func (c *APIConfig) skipUnlistedRoute(key, name string, fd layered.Found) bool { if err != nil { return false } + text, _ = routeValue(key, text) provider, model, ok := strings.Cut(text, "/") if !ok || provider == "" || model == "" { return false diff --git a/internal/core/oracle/judgement_mode_test.go b/internal/core/oracle/judgement_mode_test.go new file mode 100644 index 0000000000..3e6dcb1ac9 --- /dev/null +++ b/internal/core/oracle/judgement_mode_test.go @@ -0,0 +1,111 @@ +package oracle + +import ( + "strings" + "testing" +) + +// A judgement type's route carries a mode (itd-2609221009495079, +// spc-2609221011151661 scope 3): `host` keeps the type on the host, +// `shadow:/` runs the adapter beside the host, which still +// decides, and a bare `/` turns the type over to the adapter. +// The allowlist is judged for a shadow route exactly as for one turned on, when +// the configuration is read and before any call (criterion 4). + +func TestAJudgementRouteCarriesItsMode(t *testing.T) { + f := newFx(t) + f.machineConfig(`{"oracle":{"api":{` + openrouterBlock + `}, + "judgements":{"duplicate-match":"shadow:openrouter/typesafe/jev-1.13", + "remedy-read":"openrouter/typesafe/jev-latest", + "memory-check":"host"}}}`) + c := f.loadAPI() + if len(c.Diagnostics) != 0 { + t.Fatalf("diagnostics = %q, want none", c.Diagnostics) + } + tgt, ok := c.Judgement("duplicate-match") + if !ok || tgt.Provider != "openrouter" || tgt.Model != "typesafe/jev-1.13" || tgt.Mode != JudgementShadow { + t.Fatalf("shadow judgement = %+v, %v", tgt, ok) + } + if got := tgt.Route(); got != "shadow:openrouter/typesafe/jev-1.13" { + t.Fatalf("Route() = %q", got) + } + tgt, ok = c.Judgement("remedy-read") + if !ok || tgt.Model != "typesafe/jev-latest" || tgt.Mode != JudgementOn { + t.Fatalf("judgement turned on = %+v, %v", tgt, ok) + } + if got := tgt.Route(); got != "openrouter/typesafe/jev-latest" { + t.Fatalf("Route() = %q", got) + } + if tgt, ok := c.Judgement("memory-check"); ok { + t.Fatalf("a host judgement is pointed at %+v; host means no route", tgt) + } + // A role carries no mode: the shadow prefix is the judgements' alone. + for _, r := range c.Routes() { + if r.Kind == "role" && r.Target.Mode != "" { + t.Fatalf("role %s carries mode %q", r.Name, r.Target.Mode) + } + } +} + +func TestAShadowRouteToAnUnlistedModelIsRefusedBeforeAnyCall(t *testing.T) { + f := newFx(t) + f.machineConfig(`{"oracle":{"api":{` + openrouterBlock + `}, + "judgements":{"duplicate-match":"shadow:openrouter/mistral/small"}}}`) + err := f.loadAPIErr() + for _, want := range []string{"not on openrouter's list", "typesafe/jev-1.13, typesafe/jev-latest", "refused before any call"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("refusal %q does not name %q", err, want) + } + } +} + +func TestADenylistedShadowRouteIsRefused(t *testing.T) { + f := newFx(t) + f.machineConfig(`{"oracle":{"api":{` + openrouterBlock + `},"denylist":["typesafe/*"], + "judgements":{"duplicate-match":"shadow:openrouter/typesafe/jev-1.13"}}}`) + if err := f.loadAPIErr(); !strings.Contains(err.Error(), "typesafe/*") { + t.Fatalf("refusal = %v, want the denylist entry named", err) + } +} + +func TestARepositoryShadowRouteToAKeyedProviderIsSkipped(t *testing.T) { + f := newFx(t) + f.machineConfig(`{"oracle":{"api":{` + openrouterBlock + `}}}`) + f.repoConfig(`{"oracle":{"judgements":{"duplicate-match":"shadow:openrouter/typesafe/jev-1.13"}}}`) + c := f.loadAPI() + if tgt, ok := c.Judgement("duplicate-match"); ok { + t.Fatalf("a repository's shadow route spends the person's key: %+v", tgt) + } + if len(c.Diagnostics) != 1 || !strings.Contains(c.Diagnostics[0], "holds a key") { + t.Fatalf("diagnostics = %q, want the keyed-route skip", c.Diagnostics) + } +} + +func TestARepositoryHostJudgementOverridesTheMachineRoute(t *testing.T) { + f := newFx(t) + f.machineConfig(`{"oracle":{"api":{` + localBlock + `},"judgements":{"duplicate-match":"local/qwen/qwen3-8b"}}}`) + f.repoConfig(`{"oracle":{"judgements":{"duplicate-match":"host"}}}`) + c := f.loadAPI() + if tgt, ok := c.Judgement("duplicate-match"); ok { + t.Fatalf("judgement = %+v; the repository's host keeps it on the host", tgt) + } + if len(c.Diagnostics) != 0 { + t.Fatalf("diagnostics = %q, want none: host is not a malformed route", c.Diagnostics) + } +} + +func TestAMalformedShadowRouteIsRefusedOnTheMachine(t *testing.T) { + f := newFx(t) + f.machineConfig(`{"oracle":{"api":{` + openrouterBlock + `},"judgements":{"duplicate-match":"shadow:openrouter"}}}`) + if err := f.loadAPIErr(); !strings.Contains(err.Error(), "/") { + t.Fatalf("refusal = %v", err) + } + // A role takes no mode: shadow: on a role names a provider called + // "shadow:openrouter", which is not a plain name and not configured. + f = newFx(t) + f.machineConfig(`{"oracle":{"api":{` + openrouterBlock + `},"roles":{"scribe":"shadow:openrouter/typesafe/jev-1.13"}}}`) + c := f.loadAPI() + if tgt, ok := c.Role("scribe"); ok { + t.Fatalf("a role took the shadow mode: %+v", tgt) + } +} From ade43b0352b727f2487bee03c9736bfc892a961a Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:06:37 +0000 Subject: [PATCH 03/20] feat: record decision adapter shadow pairs in a lab and harvest their agreement A shadow pair (host answer, adapter answer, probability, state hash and size) is appended to pairs.jsonl in the newest lab home of the repository's lane, never in the repository. lab harvest reads it back as agreement per judgement type with counts, skips and four probability bands, plus the line a research note carries naming the lab home and the count. The lab chapter now describes the computed section. Resolves: iss-2610262037348735 Assisted-by: Claude:claude-opus-5-5 --- .abcd/development/brief/04-surfaces/31-lab.md | 20 ++ ...5-the-decision-adapter-intent-needs-lab.md | 9 + internal/core/lab/harvest.go | 15 +- internal/core/lab/pairs.go | 261 ++++++++++++++++++ internal/core/lab/pairs_test.go | 150 ++++++++++ internal/surface/cli/lab.go | 12 + internal/surface/cli/lab_surface_test.go | 43 +++ 7 files changed, 509 insertions(+), 1 deletion(-) rename .abcd/work/issues/{open => resolved}/iss-2610262037348735-the-decision-adapter-intent-needs-lab.md (79%) create mode 100644 internal/core/lab/pairs.go create mode 100644 internal/core/lab/pairs_test.go diff --git a/.abcd/development/brief/04-surfaces/31-lab.md b/.abcd/development/brief/04-surfaces/31-lab.md index 7ce8b9019b..160bf4cc48 100644 --- a/.abcd/development/brief/04-surfaces/31-lab.md +++ b/.abcd/development/brief/04-surfaces/31-lab.md @@ -128,6 +128,26 @@ Coverage grades each section grounded, partial or blank, the lifeboat's vocabulary. The harvest files nothing, and claims no cost the lab did not measure. +A lab that collected the decision adapter's shadow pairs +(itd-2609221009495079) gets one more section. When a judgement type is +configured for shadow (`oracle.judgements.` set to +`shadow:/`, [`03-configuration.md`](../05-internals/03-configuration.md)), +the host decides and the adapter answers beside it. Each pair is appended to +`pairs.jsonl` in the newest lab home minted for the repository: the host's +answer, the adapter's answer, the adapter's probability, the state's SHA-256 and +size, and the model that answered, or why it did not. That is the only file the +adapter writes, and it is never written into the repository. Here the harvest +computes something rather than only assembling prose: from `pairs.jsonl` it +counts, for each judgement type, the pairs, the pairs compared (the adapter +answered), the pairs that agreed and the pairs skipped (the state was larger +than the model's window, or the call failed). It reports the agreement rate, +and the compared and agreed counts in four probability bands (below 0.50, +0.50–0.70, 0.70–0.90, 0.90 and above). Lines that do not parse are counted +apart and never guessed at. The section ends with the line a research note +filed from it carries, naming the lab home and the count. That note is the +evidence a ruling turns a type on with. A lab with no pairs has no such +section. + ## Halt and record A gate refusal halts the lab and is recorded, never adapted around. When the diff --git a/.abcd/work/issues/open/iss-2610262037348735-the-decision-adapter-intent-needs-lab.md b/.abcd/work/issues/resolved/iss-2610262037348735-the-decision-adapter-intent-needs-lab.md similarity index 79% rename from .abcd/work/issues/open/iss-2610262037348735-the-decision-adapter-intent-needs-lab.md rename to .abcd/work/issues/resolved/iss-2610262037348735-the-decision-adapter-intent-needs-lab.md index 69b8a41a2e..a11d0a26bd 100644 --- a/.abcd/work/issues/open/iss-2610262037348735-the-decision-adapter-intent-needs-lab.md +++ b/.abcd/work/issues/resolved/iss-2610262037348735-the-decision-adapter-intent-needs-lab.md @@ -12,6 +12,11 @@ found_at: ".abcd/development/intents/planned/itd-2609221009495079-abcd-s-typed-j remedy: "In the decision-adapter intent state that lab harvest gains a per-type agreement section computed from the adapter's recorded pairs (new scope on the shipped lab verb), or name a new sub-verb for it; the lab chapter describes a harvest that only assembles prose sections." related_intents: [itd-2609221009495079] finding_ref: cfk-98716f3a82450774 +resolution: "lab harvest gains a per-type agreement section computed from the decision adapter's pairs.jsonl (counts, compared, agreed, skipped, rate, four probability bands, and the research note's line naming the lab home and the count); the lab chapter now describes it" +impact: internal +resolved_by: + intent: "itd-2609221009495079" + spec: "spc-2609221011151661" --- The decision-adapter intent needs lab harvest to report agreement rates with probability bands; the lab brief's harvest only assembles prose sections from findings and probe records @@ -24,3 +29,7 @@ A sequencing impossibility (severity minor) found by the cross-document consiste The intent's evidence for turning a judgement type on is the agreement rates per type, with counts and probability bands, that `abcd lab harvest` produces from pairs the adapter recorded. In the lab chapter, the harvest assembles the lifeboat's sections from the INTENTION, the findings log and the probe records, and computes no statistics. The lab verbs run nothing either, because whoever runs the lab runs each probe. As the lab is written, its harvest cannot produce the figures this intent depends on. Evidence: `.abcd/work/reviews/2026-10-26-consistency/00-summary.md`, finding 733. + +## Grounds + +- pursued: the harvest's agreement section is the evidence path the intent names; shown wrong if a lab holding pairs harvests without per-type counts and bands diff --git a/internal/core/lab/harvest.go b/internal/core/lab/harvest.go index 716cf7f2eb..c0c2e5ec9e 100644 --- a/internal/core/lab/harvest.go +++ b/internal/core/lab/harvest.go @@ -51,13 +51,20 @@ type Harvested struct { Gaps []Gap `json:"gaps"` Coverage []SectionCoverage `json:"coverage"` Halted []string `json:"halted"` + // Agreement is the decision adapter's shadow agreement over the lab's + // pairs.jsonl (pairs.go); absent when the lab recorded no pair. + Agreement *Agreement `json:"agreement,omitempty"` } // Harvest assembles the lab's harvest in the lifeboat's section shape — // intention, method, what worked, what is open, candidates, coverage — from the // INTENTION, the findings log and the probe records, citing each finding's probe // records by path. Its product findings are listed as capture candidates, each -// with the capture line that files it, found during this lab. +// with the capture line that files it, found during this lab. A lab that +// recorded the decision adapter's shadow pairs (pairs.go) also gets the +// agreement per judgement type, with counts and probability bands, and the +// line a research note filed from it carries, naming the lab home and the +// count (itd-2609221009495079 criterion 2). // // No claim outlives its input: a finding that cites no probe record or evidence // file, or cites one that is missing or incomplete, is a gap, and a harvest with @@ -132,6 +139,9 @@ func Harvest(repoRoot, id string) (Harvested, error) { } } res.Coverage = coverage(intention, findings, probes, res) + if res.Agreement, err = l.agreement(); err != nil { + return Harvested{}, err + } if len(res.Gaps) > 0 { return res, fmt.Errorf("%w: %d finding citation(s) cannot be verified from the lab's records; nothing was written", ErrHalted, len(res.Gaps)) } @@ -301,5 +311,8 @@ func harvestDoc(e Entry, intention string, findings []Finding, probes map[string fmt.Fprintf(&b, "| %s | %s | %s |\n", c.Section, c.Status, c.Note) } b.WriteString("\nCost: this harvest records what the lab's records hold and claims no token or\nmoney cost the lab did not measure.\n") + if res.Agreement != nil { + b.WriteString("\n" + agreementDoc(res.Agreement)) + } return b.String() } diff --git a/internal/core/lab/pairs.go b/internal/core/lab/pairs.go new file mode 100644 index 0000000000..526917c846 --- /dev/null +++ b/internal/core/lab/pairs.go @@ -0,0 +1,261 @@ +package lab + +// pairs.go is the decision adapter's shadow record (itd-2609221009495079, +// spc-2609221011151661 scopes 3 and 4): while a judgement type runs in shadow +// the host decides and the adapter answers beside it, and each pair (the +// host's answer, the adapter's, its probability and the state's hash) is +// appended to pairs.jsonl in a lab home, in the machine-scoped store and never +// the repository. The harvest reads the file back as the agreement report a +// research note turns a type on with. +// +// A pair goes to the newest lab minted for the repository: a lab is minted to +// answer one question, and the shadow run is the question "does the adapter +// agree with the host?", so the person mints the lab before configuring a type +// for shadow, and the lab the latest mint opened collects the pairs. + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "sort" + "strings" + + "github.com/intentdriven/abcd/internal/fsutil" +) + +// pairsName is the shadow record inside a lab home. +const pairsName = "pairs.jsonl" + +// maxPairBytes bounds one recorded line: a pair carries answers and a hash, +// never the state itself. +const maxPairBytes = 8 << 10 + +// SkippedWindow marks a pair whose state exceeded the adapter's window, so +// only the host answered; SkippedFailed one whose adapter call failed. +const ( + SkippedWindow = "window" + SkippedFailed = "adapter-failed" +) + +// ErrNoLab is RecordPair's answer when the repository has no lab to record +// in: the host has decided, and the pair is not kept. +var ErrNoLab = errors.New("no lab minted for this repository") + +// Pair is one shadow judgement: the type and its question kind, the state's +// hash and size, the host's answer, and the adapter's answer with its +// probability, or why the adapter did not answer. +type Pair struct { + At string `json:"at"` + Type string `json:"type"` + Kind string `json:"kind"` + StateSHA256 string `json:"state_sha256"` + StateTokens int `json:"state_tokens"` + Host string `json:"host"` + Adapter string `json:"adapter,omitempty"` + Probability *float64 `json:"probability,omitempty"` + // Target is the adapter's /; ModelReported the model + // the provider said answered. + Target string `json:"target"` + ModelReported string `json:"model_reported,omitempty"` + // Skipped is SkippedWindow or SkippedFailed when the adapter gave no + // answer, and Detail says why. + Skipped string `json:"skipped,omitempty"` + Detail string `json:"detail,omitempty"` +} + +// RecordPair appends p to pairs.jsonl in the newest lab minted for the +// repository at repoRoot, stamping its time, and returns the lab's id and the +// file in tilde form. It refuses a pair without its type, kind, state hash or +// host answer, and returns ErrNoLab when the repository has no lab. +func RecordPair(repoRoot string, p Pair) (id, where string, err error) { + if p.Type == "" || p.Kind == "" || len(p.StateSHA256) != 64 || p.Host == "" { + return "", "", fmt.Errorf("%w: a pair names its type, kind, state hash and the host's answer", ErrRefused) + } + s, err := resolveStore(repoRoot) + if err != nil { + return "", "", err + } + es, _, err := s.entries() + if err != nil { + return "", "", err + } + if len(es) == 0 { + return "", "", fmt.Errorf("%w in %s; mint one with `abcd lab mint` to record shadow pairs", ErrNoLab, s.display()) + } + newest := es[0] + for _, e := range es[1:] { + if e.ID > newest.ID { + newest = e + } + } + l, err := open(repoRoot, newest.ID) + if err != nil { + return "", "", err + } + defer l.close() + p.At = stamp() + line, err := json.Marshal(p) + if err != nil { + return "", "", err + } + if len(line) >= maxPairBytes { + return "", "", fmt.Errorf("%w: a pair of %d bytes exceeds the %d a line may hold", ErrRefused, len(line), maxPairBytes) + } + if fi, err := l.root.Lstat(pairsName); err == nil && !fi.Mode().IsRegular() { + return "", "", fmt.Errorf("%w: %s is not a regular file", ErrRefused, l.display(pairsName)) + } + f, err := l.root.OpenFile(pairsName, os.O_WRONLY|os.O_APPEND|os.O_CREATE, fileMode) + if err != nil { + return "", "", fmt.Errorf("cannot open %s: %v", l.display(pairsName), redact(err, l.store.home)) + } + if _, err := f.Write(append(line, '\n')); err != nil { + f.Close() + return "", "", fmt.Errorf("cannot append to %s: %v", l.display(pairsName), redact(err, l.store.home)) + } + if err := f.Close(); err != nil { + return "", "", fmt.Errorf("cannot close %s: %v", l.display(pairsName), redact(err, l.store.home)) + } + return newest.ID, l.display(pairsName), nil +} + +// Band is one probability band of a type's compared pairs. +type Band struct { + Band string `json:"band"` + Compared int `json:"compared"` + Agreed int `json:"agreed"` +} + +// TypeAgreement is one judgement type's agreement: every pair recorded, the +// ones the adapter answered (compared), those whose answers matched, those it +// did not answer (skipped), the rate over the compared, and the bands. +type TypeAgreement struct { + Type string `json:"type"` + Pairs int `json:"pairs"` + Compared int `json:"compared"` + Agreed int `json:"agreed"` + Skipped int `json:"skipped"` + Rate float64 `json:"rate"` + Bands []Band `json:"bands"` +} + +// Agreement is the harvest's report over a lab's pairs: the lab home, the +// pairs counted, the lines that did not parse, each type's agreement, and the +// line a research note filed from it carries. +type Agreement struct { + LabHome string `json:"lab_home"` + Pairs int `json:"pairs"` + Unreadable int `json:"unreadable"` + Types []TypeAgreement `json:"types"` + Note string `json:"note"` +} + +// bands are the probability bands, lowest first: [lo, hi), the last closed. +var bands = []struct { + name string + lo, hi float64 +}{{"0.00-0.50", 0, 0.5}, {"0.50-0.70", 0.5, 0.7}, {"0.70-0.90", 0.7, 0.9}, {"0.90-1.00", 0.9, 1.0000001}} + +// agreement reads the lab's pairs.jsonl; nil when the lab recorded none. +func (l *lab) agreement() (*Agreement, error) { + data, err := fsutil.ReadGuardedInRoot(l.root, pairsName, maxDocBytes) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("cannot read %s: %v", l.display(pairsName), redact(err, l.store.home)) + } + a := &Agreement{LabHome: l.store.display(l.entry.ID), Types: []TypeAgreement{}} + by := map[string]*TypeAgreement{} + sc := bufio.NewScanner(bytes.NewReader(data)) + sc.Buffer(make([]byte, 0, maxPairBytes), maxPairBytes) + for sc.Scan() { + line := bytes.TrimSpace(sc.Bytes()) + if len(line) == 0 { + continue + } + var p Pair + if json.Unmarshal(line, &p) != nil || p.Type == "" || p.Host == "" { + a.Unreadable++ + continue + } + a.Pairs++ + t := by[p.Type] + if t == nil { + t = &TypeAgreement{Type: p.Type, Bands: make([]Band, len(bands))} + for i, b := range bands { + t.Bands[i].Band = b.name + } + by[p.Type] = t + } + t.Pairs++ + if p.Skipped != "" || p.Adapter == "" || p.Probability == nil { + t.Skipped++ + continue + } + t.Compared++ + agreed := p.Adapter == p.Host + if agreed { + t.Agreed++ + } + for i, b := range bands { + if *p.Probability >= b.lo && *p.Probability < b.hi { + t.Bands[i].Compared++ + if agreed { + t.Bands[i].Agreed++ + } + } + } + } + if err := sc.Err(); err != nil { + return nil, fmt.Errorf("cannot read %s: %v", l.display(pairsName), err) + } + names := make([]string, 0, len(by)) + for n := range by { + names = append(names, n) + } + sort.Strings(names) + compared := 0 + for _, n := range names { + t := by[n] + if t.Compared > 0 { + t.Rate = float64(t.Agreed) / float64(t.Compared) + } + compared += t.Compared + a.Types = append(a.Types, *t) + } + a.Note = fmt.Sprintf("Shadow agreement from the lab home %s: %d pairs (%d compared) over %d judgement type(s), harvested %s.", + a.LabHome, a.Pairs, compared, len(a.Types), stamp()) + return a, nil +} + +// agreementDoc renders the harvest's agreement section. +func agreementDoc(a *Agreement) string { + var b strings.Builder + b.WriteString("## Agreement — decision adapter shadow pairs\n\n") + fmt.Fprintf(&b, "From `%s/%s`: %d pairs", a.LabHome, pairsName, a.Pairs) + if a.Unreadable > 0 { + fmt.Fprintf(&b, ", %d line(s) that did not parse and are not counted", a.Unreadable) + } + b.WriteString(". A pair is compared when the adapter answered; one it did not answer (its state\nexceeded the window, or the call failed) is skipped, and the host decided alone.\n\n") + b.WriteString("| Type | Pairs | Compared | Agreed | Skipped | Agreement |\n| --- | --- | --- | --- | --- | --- |\n") + for _, t := range a.Types { + rate := "n/a" + if t.Compared > 0 { + rate = fmt.Sprintf("%.0f%%", t.Rate*100) + } + fmt.Fprintf(&b, "| %s | %d | %d | %d | %d | %s |\n", t.Type, t.Pairs, t.Compared, t.Agreed, t.Skipped, rate) + } + b.WriteString("\nBy the adapter's probability (agreed/compared):\n\n") + for _, t := range a.Types { + parts := make([]string, 0, len(t.Bands)) + for _, bd := range t.Bands { + parts = append(parts, fmt.Sprintf("%s: %d/%d", bd.Band, bd.Agreed, bd.Compared)) + } + fmt.Fprintf(&b, "- %s — %s\n", t.Type, strings.Join(parts, ", ")) + } + fmt.Fprintf(&b, "\nFor the research note that turns a type on:\n\n> %s\n", a.Note) + return b.String() +} diff --git a/internal/core/lab/pairs_test.go b/internal/core/lab/pairs_test.go new file mode 100644 index 0000000000..0ce09349fc --- /dev/null +++ b/internal/core/lab/pairs_test.go @@ -0,0 +1,150 @@ +package lab + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func prob(p float64) *float64 { return &p } + +// TestRecordPairWritesUnderTheNewestLabHomeAndNothingInTheRepo is the +// decision adapter's first criterion, the lab's half (itd-2609221009495079): +// a shadow pair is appended to pairs.jsonl in the lab home, in the +// machine-scoped store, and nothing is written into the repository. +func TestRecordPairWritesUnderTheNewestLabHomeAndNothingInTheRepo(t *testing.T) { + r, home := fixture(t) + notAbcd(t) + m := mint(t, r) + before := r.Git("status", "--porcelain", "--ignored") + + p := Pair{Type: "duplicate-match", Kind: "choose", StateSHA256: strings.Repeat("a", 64), StateTokens: 12, + Host: "iss-1", Adapter: "iss-1", Probability: prob(0.92), Target: "openrouter/typesafe/jev-1.13"} + id, where, err := RecordPair(r.Root(), p) + if err != nil { + t.Fatalf("RecordPair: %v", err) + } + if id != m.ID || !strings.HasSuffix(where, m.ID+"/pairs.jsonl") || strings.Contains(where, home) { + t.Fatalf("recorded in %s at %s, want lab %s in tilde form", id, where, m.ID) + } + if _, _, err := RecordPair(r.Root(), Pair{Type: "duplicate-match", Kind: "choose", StateSHA256: strings.Repeat("b", 64), + StateTokens: 40000, Host: "none", Target: "openrouter/typesafe/jev-1.13", Skipped: SkippedWindow}); err != nil { + t.Fatal(err) + } + lines := strings.Split(strings.TrimSpace(read(t, filepath.Join(labDir(home, m), "pairs.jsonl"))), "\n") + if len(lines) != 2 { + t.Fatalf("pairs.jsonl holds %d lines, want 2", len(lines)) + } + var got Pair + if err := json.Unmarshal([]byte(lines[0]), &got); err != nil || got.Host != "iss-1" || got.Probability == nil || + *got.Probability != 0.92 || got.At == "" { + t.Fatalf("first pair = %+v, %v", got, err) + } + if fi, err := os.Stat(filepath.Join(labDir(home, m), "pairs.jsonl")); err != nil || fi.Mode().Perm() != 0o600 { + t.Fatalf("pairs.jsonl mode = %v, %v; want 0600", fi, err) + } + if after := r.Git("status", "--porcelain", "--ignored"); after != before { + t.Fatalf("the repository changed:\n%s\nwas\n%s", after, before) + } +} + +func TestRecordPairWithNoLabRefuses(t *testing.T) { + r, _ := fixture(t) + _, _, err := RecordPair(r.Root(), Pair{Type: "duplicate-match", Kind: "choose", StateSHA256: strings.Repeat("a", 64), Host: "x"}) + if !errors.Is(err, ErrNoLab) { + t.Fatalf("err = %v, want ErrNoLab", err) + } +} + +func TestRecordPairRefusesAPairWithoutItsType(t *testing.T) { + r, _ := fixture(t) + notAbcd(t) + mint(t, r) + if _, _, err := RecordPair(r.Root(), Pair{Kind: "choose", StateSHA256: strings.Repeat("a", 64), Host: "x"}); !errors.Is(err, ErrRefused) { + t.Fatalf("err = %v, want a refusal", err) + } +} + +// TestHarvestReportsAgreementPerTypeWithBands is the second criterion: the +// harvest reports agreement per judgement type with counts and probability +// bands, and the note it renders names the lab home and the count. +func TestHarvestReportsAgreementPerTypeWithBands(t *testing.T) { + r, home := fixture(t) + notAbcd(t) + m := mint(t, r) + sha := strings.Repeat("c", 64) + for _, p := range []Pair{ + {Type: "duplicate-match", Host: "a", Adapter: "a", Probability: prob(0.95)}, + {Type: "duplicate-match", Host: "a", Adapter: "b", Probability: prob(0.95)}, + {Type: "duplicate-match", Host: "b", Adapter: "b", Probability: prob(0.6)}, + {Type: "duplicate-match", Host: "b", Skipped: SkippedWindow}, + {Type: "remedy-read", Host: "yes", Adapter: "yes", Probability: prob(0.3)}, + } { + p.Kind, p.StateSHA256, p.Target = "choose", sha, "openrouter/typesafe/jev-1.13" + if _, _, err := RecordPair(r.Root(), p); err != nil { + t.Fatal(err) + } + } + // A line that does not parse is counted, never guessed at. + f, err := os.OpenFile(filepath.Join(labDir(home, m), "pairs.jsonl"), os.O_APPEND|os.O_WRONLY, 0) + if err != nil { + t.Fatal(err) + } + f.WriteString("{not json\n") + f.Close() + + res, err := Harvest(r.Root(), m.ID) + if err != nil { + t.Fatalf("Harvest: %v", err) + } + a := res.Agreement + if a == nil || a.Pairs != 5 || a.Unreadable != 1 || len(a.Types) != 2 { + t.Fatalf("agreement = %+v", a) + } + dm := a.Types[0] + if dm.Type != "duplicate-match" || dm.Pairs != 4 || dm.Compared != 3 || dm.Agreed != 2 || dm.Skipped != 1 { + t.Fatalf("duplicate-match = %+v", dm) + } + bands := map[string]Band{} + for _, b := range dm.Bands { + bands[b.Band] = b + } + if b := bands["0.90-1.00"]; b.Compared != 2 || b.Agreed != 1 { + t.Fatalf("0.90-1.00 band = %+v (bands %+v)", b, dm.Bands) + } + if b := bands["0.50-0.70"]; b.Compared != 1 || b.Agreed != 1 { + t.Fatalf("0.50-0.70 band = %+v", b) + } + if rr := a.Types[1]; rr.Type != "remedy-read" || rr.Compared != 1 || rr.Agreed != 1 { + t.Fatalf("remedy-read = %+v", rr) + } + labHome := StoreDisplay + "/" + m.RootSHA + "/" + m.ID + if a.LabHome != labHome || !strings.Contains(a.Note, labHome) || !strings.Contains(a.Note, "5 pairs") { + t.Fatalf("note = %q, home = %q; want the lab home %s and the count", a.Note, a.LabHome, labHome) + } + doc := read(t, filepath.Join(labDir(home, m), "harvest", "harvest.md")) + for _, want := range []string{"## Agreement", "| duplicate-match | 4 | 3 | 2 | 1 | 67% |", "0.90-1.00: 1/2", a.Note} { + if !strings.Contains(doc, want) { + t.Errorf("harvest.md lacks %q:\n%s", want, doc) + } + } +} + +func TestHarvestWithNoPairsHasNoAgreement(t *testing.T) { + r, home := fixture(t) + notAbcd(t) + m := mint(t, r) + res, err := Harvest(r.Root(), m.ID) + if err != nil { + t.Fatal(err) + } + if res.Agreement != nil { + t.Fatalf("agreement = %+v, want none without pairs", res.Agreement) + } + if strings.Contains(read(t, filepath.Join(labDir(home, m), "harvest", "harvest.md")), "## Agreement") { + t.Fatal("harvest.md carries an agreement section with no pairs recorded") + } +} diff --git a/internal/surface/cli/lab.go b/internal/surface/cli/lab.go index a0c9e59710..450a158dc0 100644 --- a/internal/surface/cli/lab.go +++ b/internal/surface/cli/lab.go @@ -357,4 +357,16 @@ func renderHarvest(w io.Writer, res lab.Harvested) { fmt.Fprintf(w, " %s %s\n", c.Finding, termsafe.Sanitize(c.Command)) } } + // The decision adapter's shadow agreement (itd-2609221009495079): one + // line per judgement type, then the research note's line. + if a := res.Agreement; a != nil { + for _, t := range a.Types { + fmt.Fprintf(w, "agreement %s: %d/%d compared agree (%.0f%%), %d skipped\n", + termsafe.Sanitize(t.Type), t.Agreed, t.Compared, t.Rate*100, t.Skipped) + } + if a.Unreadable > 0 { + fmt.Fprintf(w, " %s in pairs.jsonl did not parse and are not counted\n", countOf(a.Unreadable, "line")) + } + fmt.Fprintf(w, "note: %s\n", termsafe.Sanitize(a.Note)) + } } diff --git a/internal/surface/cli/lab_surface_test.go b/internal/surface/cli/lab_surface_test.go index a86fd05357..ba00938662 100644 --- a/internal/surface/cli/lab_surface_test.go +++ b/internal/surface/cli/lab_surface_test.go @@ -148,3 +148,46 @@ func TestLabVerbRefusesARequestItCannotServe(t *testing.T) { t.Errorf("lab outside a checkout: exit %d, %q", code, errOut) } } + +// The harvest's agreement report reaches the CLI (itd-2609221009495079 +// criterion 2): a lab holding decision-adapter shadow pairs prints the +// agreement per type and the research note's line, naming the lab home and the +// count, in the text and in --json, with no home path in either. +func TestLabHarvestPrintsTheShadowAgreement(t *testing.T) { + home, repo := labCheckout(t) + code, out, errOut := runLab(t, "lab", "--json", "mint", "does", "the", "adapter", "agree?") + if code != 0 { + t.Fatalf("lab mint exit %d: %s%s", code, out, errOut) + } + var m struct { + ID string `json:"id"` + } + if err := json.Unmarshal([]byte(out), &m); err != nil { + t.Fatal(err) + } + p := 0.9 + for _, adapter := range []string{"no", "yes"} { + if _, _, err := lab.RecordPair(repo, lab.Pair{Type: "eligibility-residual", Kind: "choose", StateSHA256: strings.Repeat("d", 64), + Host: "no", Adapter: adapter, Probability: &p, Target: "openrouter/typesafe/jev-1.13"}); err != nil { + t.Fatal(err) + } + } + code, out, _ = runLab(t, "lab", "harvest", m.ID) + if code != 0 || !strings.Contains(out, "agreement eligibility-residual: 1/2 compared agree (50%), 0 skipped") || + !strings.Contains(out, "2 pairs (2 compared)") || !strings.Contains(out, m.ID) { + t.Fatalf("harvest (exit %d):\n%s", code, out) + } + code, out, _ = runLab(t, "lab", "--json", "harvest", m.ID) + var res struct { + Agreement struct { + Pairs int `json:"pairs"` + Note string `json:"note"` + } `json:"agreement"` + } + if code != 0 || json.Unmarshal([]byte(out), &res) != nil || res.Agreement.Pairs != 2 || !strings.Contains(res.Agreement.Note, m.ID) { + t.Fatalf("harvest --json (exit %d):\n%s", code, out) + } + if strings.Contains(out, home) { + t.Fatalf("harvest --json leaked the home path:\n%s", out) + } +} From 6b8f11bd0ab975fc1ce2bc9c2042dd5c5d20d56a Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:18:47 +0000 Subject: [PATCH 04/20] feat: route typed judgements through one interface, host or decision adapter The judge package is the decision adapter's interface: a typed question (choose, score, yes/no) over a state, answered with a value and a probability. The host and the API adapter both implement it, and a router switches each judgement type by configuration alone: host; shadow, where the host decides and the pair is recorded in the newest lab home, skipped with its size when the state exceeds the window; or on, where the adapter decides and an adapter that cannot answer falls back to the host with a receipt. The allowlist is judged when the configuration is read, before any call. Assisted-by: Claude:claude-opus-5-5 --- internal/README.md | 12 + internal/core/decide/judge/adapter.go | 101 ++++++ internal/core/decide/judge/host.go | 52 +++ internal/core/decide/judge/judge.go | 218 +++++++++++++ internal/core/decide/judge/judge_test.go | 392 +++++++++++++++++++++++ internal/core/decide/judge/router.go | 264 +++++++++++++++ 6 files changed, 1039 insertions(+) create mode 100644 internal/core/decide/judge/adapter.go create mode 100644 internal/core/decide/judge/host.go create mode 100644 internal/core/decide/judge/judge.go create mode 100644 internal/core/decide/judge/judge_test.go create mode 100644 internal/core/decide/judge/router.go diff --git a/internal/README.md b/internal/README.md index 15a39168b7..dff48ed46f 100644 --- a/internal/README.md +++ b/internal/README.md @@ -237,6 +237,18 @@ plugin surface, and a future MCP server share one engine. already resolves. It owns the id, the date, the filename and the four sections, and states nothing: the decision is a human's to write, so the record lands `proposed` and the skeleton carries questions rather than answers. +- **`core/decide/judge/`** — the decision adapter's one interface + (itd-2609221009495079), apart from the ADR verb in spite of the shared name: a + typed judgement, (state, question) → (value, probability), whose question is + one of three kinds (choose, score, yes/no) and whose type is one of a closed + registry. The host's judgement (`Host`, probability 1) and the decision model + reached through `core/oracle`'s API call (`Adapter`) both implement it. A + `Router` reads `oracle.judgements.` as `host`, `shadow:/` + or `/`, which the oracle reader has already held to the + provider's allowlist. In shadow the host decides and the pair is recorded in + the newest lab home's `pairs.jsonl` (`core/lab`). Turned on, the adapter + decides, and an adapter that cannot answer leaves the decision to the host + with a receipt. - **`core/decisions/`** — the decision records' folder, `.abcd/work/decisions/`: one `dcn--.md` file per decision (adr-2610110925465593). `Record` is its one writer, behind `abcd decide ""`: it mints through diff --git a/internal/core/decide/judge/adapter.go b/internal/core/decide/judge/adapter.go new file mode 100644 index 0000000000..b59ce73b91 --- /dev/null +++ b/internal/core/decide/judge/adapter.go @@ -0,0 +1,101 @@ +package judge + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "math" + "strings" + + "github.com/intentdriven/abcd/internal/adapter/openaiapi" + "github.com/intentdriven/abcd/internal/core/credential" + "github.com/intentdriven/abcd/internal/core/oracle" + "github.com/intentdriven/abcd/internal/termsafe" +) + +// Caller is one call through the OpenAI-compatible API adapter; +// *oracle.APIConfig implements it. Its Call admits the target against the +// allowlist and oracle.denylist again and resolves the key by name, so a +// target built anywhere is held to adr-2609221009491186 before any call. +type Caller interface { + Call(ctx context.Context, creds credential.Source, req oracle.CallRequest, opts ...openaiapi.Option) ([]byte, oracle.CallRecord, error) +} + +// Adapter is the decision model through the one interface: the question is +// rendered as the host's is, sent to Target through Calls, and the answer is +// read as a value and a probability. The model cannot abstain, so an answer +// outside the question's choices, or a probability outside 0 to 1, is no +// answer at all. +type Adapter struct { + Calls Caller + Creds credential.Source + Target oracle.Target + Options []openaiapi.Option +} + +// adapterInstructions frame every typed question the adapter is sent. +const adapterInstructions = "You answer one typed judgement. Read the question and the state, then reply with " + + "a single JSON object and nothing else: {\"value\": \"\", " + + "\"probability\": }." + +// maxAnswerBytes bounds the answer the adapter parses: a value and a number. +const maxAnswerBytes = 4 << 10 + +// typed is the answer's shape. +type typed struct { + Value *string `json:"value"` + Probability *float64 `json:"probability"` +} + +// Judge sends the question through the API adapter and parses the answer. +func (a Adapter) Judge(ctx context.Context, s State, q Question) (Answer, error) { + if err := q.Check(); err != nil { + return Answer{}, err + } + if a.Calls == nil { + return Answer{}, fmt.Errorf("judge: no adapter configured for %s", q.Type) + } + var got Answer + contract := func(content []byte) error { + v, err := parseTyped(q, content) + got = v + return err + } + brief := openaiapi.Brief{Instructions: adapterInstructions, Input: Render(s, q)} + _, rec, err := a.Calls.Call(ctx, a.Creds, oracle.CallRequest{Target: a.Target, Brief: brief, Contract: contract}, a.Options...) + if err != nil { + return Answer{}, err + } + got.Model = rec.ModelReported + return got, nil +} + +// parseTyped reads the adapter's answer: one JSON object, perhaps inside a +// code fence, whose value is one of the question's choices and whose +// probability is a number from 0 to 1. +func parseTyped(q Question, content []byte) (Answer, error) { + if len(content) > maxAnswerBytes { + return Answer{}, fmt.Errorf("judge: the adapter's answer is %d bytes, more than the %d a typed answer takes", len(content), maxAnswerBytes) + } + body := bytes.TrimSpace(content) + if i, j := bytes.IndexByte(body, '{'), bytes.LastIndexByte(body, '}'); i >= 0 && j > i { + body = body[i : j+1] + } + var t typed + if err := json.Unmarshal(body, &t); err != nil { + return Answer{}, fmt.Errorf("judge: the adapter's answer is not a typed answer: %s", termsafe.Sanitize(err.Error())) + } + if t.Value == nil || t.Probability == nil { + return Answer{}, fmt.Errorf("judge: the adapter's answer lacks its value or its probability") + } + p := *t.Probability + if math.IsNaN(p) || p < 0 || p > 1 { + return Answer{}, fmt.Errorf("judge: the adapter's probability %v is not from 0 to 1", p) + } + v, ok := q.match(*t.Value) + if !ok { + return Answer{}, fmt.Errorf("judge: the adapter's answer %q is not one of %s", termsafe.Sanitize(clip(*t.Value)), strings.Join(q.Choices(), ", ")) + } + return Answer{Value: v, Probability: p}, nil +} diff --git a/internal/core/decide/judge/host.go b/internal/core/decide/judge/host.go new file mode 100644 index 0000000000..a48719ae4a --- /dev/null +++ b/internal/core/decide/judge/host.go @@ -0,0 +1,52 @@ +package judge + +import ( + "context" + "errors" + "fmt" + + "github.com/intentdriven/abcd/internal/termsafe" +) + +// ErrHostPending is a host judgement asked out of band: the caller has +// written the host's request (the drain's request file) and the answer comes +// back in a later move. A Router passes it through, with its receipt. +var ErrHostPending = errors.New("judge: the host's answer is awaited") + +// Host is the host's own judgement through the one interface (adr-25): Ask is +// shown the rendered prompt, the same text the adapter is sent, and returns +// the host's answer, which is parsed against the question's choices at +// probability 1. A caller whose host has already answered uses Answered. +type Host struct { + Ask func(ctx context.Context, prompt string) (string, error) +} + +// Answered is a Host whose answer is already in hand: the drain's host has +// written its answer file, or a page's host has handed its judgement back. +func Answered(text string) Host { + return Host{Ask: func(context.Context, string) (string, error) { return text, nil }} +} + +// Pending is a Host asked out of band: its Judge returns ErrHostPending. +func Pending() Host { + return Host{Ask: func(context.Context, string) (string, error) { return "", ErrHostPending }} +} + +// Judge asks the host and parses its answer at probability 1. +func (h Host) Judge(ctx context.Context, s State, q Question) (Answer, error) { + if err := q.Check(); err != nil { + return Answer{}, err + } + if h.Ask == nil { + return Answer{}, ErrHostPending + } + text, err := h.Ask(ctx, Render(s, q)) + if err != nil { + return Answer{}, err + } + v, ok := q.match(text) + if !ok { + return Answer{}, fmt.Errorf("judge: the host's answer %q to %s is not one of %v", termsafe.Sanitize(clip(text)), q.Type, q.Choices()) + } + return Answer{Value: v, Probability: 1}, nil +} diff --git a/internal/core/decide/judge/judge.go b/internal/core/decide/judge/judge.go new file mode 100644 index 0000000000..9184823894 --- /dev/null +++ b/internal/core/decide/judge/judge.go @@ -0,0 +1,218 @@ +// Package judge is the decision adapter's one interface (itd-2609221009495079, +// spc-2609221011151661 scope 1): a typed judgement, (state, question) → +// (value, probability), with three question kinds — choose one of N, score on +// a scale, yes or no. The host's own judgement implements it (Host, at +// probability 1), and so does the decision model reached through the +// OpenAI-compatible API adapter (Adapter), so the two are compared like for +// like and a judgement type is switched between them by configuration alone +// (router.go), never by its caller. +// +// It sits apart from internal/core/decide, the ADR verb's package: the name is +// shared, the concern is not. +// +// Like the rest of the core it never prints. It writes only through the +// recorder its Router is given, which is the lab's pairs file in the +// machine-scoped store (internal/core/lab), never the repository. +package judge + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "slices" + "strconv" + "strings" +) + +// Kind is a question's kind. +type Kind string + +// The three question kinds. +const ( + // KindChoose asks for one of the question's options. + KindChoose Kind = "choose" + // KindScore asks for an integer on the question's scale, Min to Max. + KindScore Kind = "score" + // KindYesNo asks yes or no. + KindYesNo Kind = "yes-no" +) + +// Type is a judgement type: one judgement-shaped step abcd may route through +// the adapter. The registry is closed: a type outside it is refused. +type Type string + +// The judgement types the intent's scope lists. Embark's lesson ranking is not +// one: the embark page ranks lessons by term overlap, a computation with no +// host answer to pair (iss-2610262033055327). +const ( + RemedyRead Type = "remedy-read" + DuplicateMatch Type = "duplicate-match" + UnmeetableCriteria Type = "unmeetable-criteria" + EligibilityResidual Type = "eligibility-residual" + SourceClassification Type = "source-classification" + SourceSelection Type = "source-selection" + MemoryCheck Type = "memory-check" + LedgerCategory Type = "ledger-category" + LedgerSeverity Type = "ledger-severity" + PrepassOverlap Type = "prepass-overlap" + PresentTense Type = "present-tense-advisory" +) + +var registry = []Type{RemedyRead, DuplicateMatch, UnmeetableCriteria, EligibilityResidual, SourceClassification, + SourceSelection, MemoryCheck, LedgerCategory, LedgerSeverity, PrepassOverlap, PresentTense} + +// Types returns the registry, in its order. +func Types() []Type { return slices.Clone(registry) } + +// Known reports whether name is a type in the registry. +func Known(name string) bool { return slices.Contains(registry, Type(name)) } + +// Bounds on a question. +const ( + maxOptions = 64 + maxOption = 200 + maxAsk = 2000 + maxScaleLen = 100 +) + +// Question is one typed question. +type Question struct { + Type Type + Kind Kind + // Ask is the question in words. + Ask string + // Options are a choose question's options. + Options []string + // Min and Max are a score question's scale, inclusive. + Min, Max int +} + +// Choices are the answers the question admits, as an answer spells them. +func (q Question) Choices() []string { + switch q.Kind { + case KindYesNo: + return []string{"yes", "no"} + case KindScore: + out := make([]string, 0, q.Max-q.Min+1) + for v := q.Min; v <= q.Max; v++ { + out = append(out, strconv.Itoa(v)) + } + return out + default: + return slices.Clone(q.Options) + } +} + +// Check refuses a question that does not hold together: a type outside the +// registry, an unknown kind, no words, or options or a scale out of shape. +func (q Question) Check() error { + if !Known(string(q.Type)) { + return fmt.Errorf("judge: %q is not a judgement type (the registry is %s)", clip(string(q.Type)), typeNames()) + } + if strings.TrimSpace(q.Ask) == "" || len(q.Ask) > maxAsk { + return fmt.Errorf("judge: a %s question needs its words (at most %d bytes)", q.Type, maxAsk) + } + switch q.Kind { + case KindYesNo: + case KindScore: + if q.Max <= q.Min || q.Max-q.Min >= maxScaleLen { + return fmt.Errorf("judge: a %s score question needs a scale of 2 to %d points, not %d to %d", q.Type, maxScaleLen, q.Min, q.Max) + } + case KindChoose: + if len(q.Options) < 1 || len(q.Options) > maxOptions { + return fmt.Errorf("judge: a %s choose question needs 1 to %d options, not %d", q.Type, maxOptions, len(q.Options)) + } + seen := map[string]bool{} + for _, o := range q.Options { + k := strings.ToLower(strings.TrimSpace(o)) + if k == "" || len(o) > maxOption || strings.ContainsAny(o, "\n\r") || seen[k] { + return fmt.Errorf("judge: a %s choose question's options are distinct one-line words of at most %d bytes", q.Type, maxOption) + } + seen[k] = true + } + default: + return fmt.Errorf("judge: %q is not a question kind (choose, score, yes-no)", clip(string(q.Kind))) + } + return nil +} + +// match returns the choice text names, matched without regard to case or +// surrounding space, and whether it is one. +func (q Question) match(text string) (string, bool) { + t := strings.TrimSpace(text) + for _, c := range q.Choices() { + if strings.EqualFold(c, t) { + return c, true + } + } + return "", false +} + +// State is what a judgement is made over, after retrieval: the text both +// implementations are shown. +type State struct { + Text string +} + +// SHA256 is the state's hash, the identity a pair records instead of the text. +func (s State) SHA256() string { + sum := sha256.Sum256([]byte(s.Text)) + return hex.EncodeToString(sum[:]) +} + +// Answer is a judgement's answer: one of the question's choices, the +// probability the answering side gives it, and, from the adapter, the model +// the provider reported. +type Answer struct { + Value string + Probability float64 + Model string +} + +// Judge is the one interface: the host and the adapter both implement it. +type Judge interface { + Judge(ctx context.Context, s State, q Question) (Answer, error) +} + +// Render is the prompt a judgement is asked as, the same for the host and the +// adapter. +func Render(s State, q Question) string { + var b strings.Builder + fmt.Fprintf(&b, "Judgement: %s (%s)\n\nQuestion: %s\n\n", q.Type, q.Kind, strings.TrimSpace(q.Ask)) + switch q.Kind { + case KindChoose: + b.WriteString("Answer with exactly one of these options:\n") + for _, o := range q.Options { + fmt.Fprintf(&b, "- %s\n", o) + } + case KindScore: + fmt.Fprintf(&b, "Answer with one whole number from %d to %d.\n", q.Min, q.Max) + case KindYesNo: + b.WriteString("Answer yes or no.\n") + } + fmt.Fprintf(&b, "\nState:\n\n%s\n", s.Text) + return b.String() +} + +// tokens estimates the tokens text takes, high rather than low: three bytes a +// token, where English prose runs nearer four, so a state that fits by this +// count fits the model's window. +func tokens(text string) int { return (len(text) + 2) / 3 } + +func typeNames() string { + out := make([]string, len(registry)) + for i, t := range registry { + out[i] = string(t) + } + return strings.Join(out, ", ") +} + +// clip bounds a caller-supplied string for a message. +func clip(s string) string { + const n = 64 + if len(s) > n { + return s[:n] + "…" + } + return s +} diff --git a/internal/core/decide/judge/judge_test.go b/internal/core/decide/judge/judge_test.go new file mode 100644 index 0000000000..26cbcd9f24 --- /dev/null +++ b/internal/core/decide/judge/judge_test.go @@ -0,0 +1,392 @@ +package judge + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/intentdriven/abcd/internal/abcdhome" + "github.com/intentdriven/abcd/internal/adapter/openaiapi" + "github.com/intentdriven/abcd/internal/core/credential" + "github.com/intentdriven/abcd/internal/core/lab" + "github.com/intentdriven/abcd/internal/core/layered" + "github.com/intentdriven/abcd/internal/core/oracle" + "github.com/intentdriven/abcd/internal/gittest" +) + +// The decision adapter (itd-2609221009495079, spc-2609221011151661): one +// interface the host and the adapter both implement, and one router with three +// modes per judgement type. The adapter here is a fake answering through the +// same interface; TestTheAPIAdapterParsesTheTypedAnswer drives the real +// Adapter over a fake oracle caller. + +var target = oracle.Target{Provider: "openrouter", Model: "typesafe/jev-1.13", Origin: "~/.abcd.noindex/config.json"} + +func dupQuestion() Question { + return Question{Type: DuplicateMatch, Kind: KindChoose, Ask: "Which record does this capture duplicate?", + Options: []string{"none", "iss-1", "iss-2"}} +} + +// fake is a Judge that answers value at p, or fails with err, and counts calls. +type fake struct { + value string + p float64 + err error + calls int + seen []string +} + +func (f *fake) Judge(_ context.Context, s State, q Question) (Answer, error) { + f.calls++ + f.seen = append(f.seen, Render(s, q)) + if f.err != nil { + return Answer{}, f.err + } + return Answer{Value: f.value, Probability: f.p, Model: "typesafe/jev-1.13-0901"}, nil +} + +// pairs is a recorder that keeps what it is given. +type pairs struct { + got []lab.Pair + err error +} + +func (p *pairs) record(pr lab.Pair) (string, string, error) { + if p.err != nil { + return "", "", p.err + } + p.got = append(p.got, pr) + return "lab-261029000000-abcdef0", "~/.abcd.noindex/lab/x/lab-261029000000-abcdef0/pairs.jsonl", nil +} + +func router(mode string, adapter *fake, rec *pairs) *Router { + routes := map[Type]oracle.Target{} + if mode != oracle.JudgementHost { + t := target + t.Mode = mode + routes[DuplicateMatch] = t + } + return New(routes, func(oracle.Target) Judge { return adapter }, rec.record) +} + +// Criterion 1: shadow — the host decides, the adapter answers beside it, and +// the pair with its probability and the state's hash is recorded. +func TestShadowTheHostDecidesAndThePairIsRecorded(t *testing.T) { + adapter, rec := &fake{value: "iss-2", p: 0.81}, &pairs{} + host := Answered("iss-1") + s := State{Text: "capture: the parser drops a trailing comma"} + a, rc, err := router(oracle.JudgementShadow, adapter, rec).Decide(context.Background(), s, dupQuestion(), host) + if err != nil { + t.Fatal(err) + } + if a.Value != "iss-1" || a.Probability != 1 || rc.Route != RouteHost || rc.Mode != oracle.JudgementShadow { + t.Fatalf("answer %+v receipt %+v: the host decides in shadow", a, rc) + } + if adapter.calls != 1 || len(rec.got) != 1 { + t.Fatalf("adapter calls %d, pairs %d; want one each", adapter.calls, len(rec.got)) + } + p := rec.got[0] + if p.Type != string(DuplicateMatch) || p.Kind != string(KindChoose) || p.Host != "iss-1" || p.Adapter != "iss-2" || + p.Probability == nil || *p.Probability != 0.81 || p.StateSHA256 != s.SHA256() || p.Target != "openrouter/typesafe/jev-1.13" || + p.ModelReported != "typesafe/jev-1.13-0901" || p.Skipped != "" { + t.Fatalf("pair = %+v", p) + } + if !strings.Contains(rc.Pair, "pairs.jsonl") { + t.Fatalf("receipt pair = %q", rc.Pair) + } +} + +// Criterion 1, end to end: the pair lands in pairs.jsonl under the newest lab +// home in the machine-scoped store, and nothing is written into the +// repository. +func TestShadowWritesThePairUnderTheLabHomeAndNothingInTheRepository(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + r := gittest.NewRepo(t) + r.Write("README.md", "root\n") + r.Commit("root") + t.Cleanup(lab.SetAbcdRootCommitForTest(strings.Repeat("a", 40))) + m, err := lab.Mint(r.Root(), "does the decision adapter agree with the host?", "") + if err != nil { + t.Fatal(err) + } + before := r.Git("status", "--porcelain", "--ignored") + t2 := target + t2.Mode = oracle.JudgementShadow + rt := New(map[Type]oracle.Target{DuplicateMatch: t2}, func(oracle.Target) Judge { return &fake{value: "none", p: 0.7} }, + func(p lab.Pair) (string, string, error) { return lab.RecordPair(r.Root(), p) }) + if _, _, err := rt.Decide(context.Background(), State{Text: "s"}, dupQuestion(), Answered("none")); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(abcdhome.Path(home, "lab", m.RootSHA, m.ID, "pairs.jsonl")) + if err != nil { + t.Fatalf("pairs.jsonl: %v", err) + } + var p lab.Pair + if err := json.Unmarshal(data, &p); err != nil || p.Host != "none" || p.Adapter != "none" { + t.Fatalf("pair = %s (%v)", data, err) + } + if after := r.Git("status", "--porcelain", "--ignored"); after != before { + t.Fatalf("the repository changed:\n%s", after) + } +} + +// Criterion 5: a state larger than the window is recorded as skipped with its +// size, the adapter is never called, and the host decides alone. +func TestShadowSkipsAStateLargerThanTheWindow(t *testing.T) { + adapter, rec := &fake{value: "iss-2", p: 0.9}, &pairs{} + rt := router(oracle.JudgementShadow, adapter, rec) + rt.Window = 100 + s := State{Text: strings.Repeat("word ", 400)} + a, rc, err := rt.Decide(context.Background(), s, dupQuestion(), Answered("iss-1")) + if err != nil || a.Value != "iss-1" || rc.Route != RouteHost { + t.Fatalf("answer %+v, receipt %+v, err %v", a, rc, err) + } + if adapter.calls != 0 { + t.Fatal("the adapter was called on a state larger than its window") + } + if len(rec.got) != 1 || rec.got[0].Skipped != lab.SkippedWindow || rec.got[0].StateTokens <= 100 || + !strings.Contains(rec.got[0].Detail, "window of 100") || rec.got[0].Adapter != "" { + t.Fatalf("pairs = %+v", rec.got) + } + if rc.PairSkipped != lab.SkippedWindow || rc.StateTokens != rec.got[0].StateTokens { + t.Fatalf("receipt = %+v", rc) + } +} + +// A shadow adapter that fails changes nothing for the host's decision; the +// pair is recorded as skipped, naming why. +func TestShadowRecordsAFailedAdapterAndTheHostStillDecides(t *testing.T) { + adapter, rec := &fake{err: errors.New("connection refused")}, &pairs{} + a, _, err := router(oracle.JudgementShadow, adapter, rec).Decide(context.Background(), State{Text: "s"}, dupQuestion(), Answered("iss-2")) + if err != nil || a.Value != "iss-2" { + t.Fatalf("answer %+v, err %v", a, err) + } + if len(rec.got) != 1 || rec.got[0].Skipped != lab.SkippedFailed || !strings.Contains(rec.got[0].Detail, "connection refused") { + t.Fatalf("pairs = %+v", rec.got) + } +} + +// No lab minted: the host decides, and the receipt says the pair was not kept. +func TestShadowWithNoLabStillDecidesAndSaysSo(t *testing.T) { + rec := &pairs{err: lab.ErrNoLab} + a, rc, err := router(oracle.JudgementShadow, &fake{value: "none", p: 0.5}, rec).Decide(context.Background(), State{Text: "s"}, dupQuestion(), Answered("none")) + if err != nil || a.Value != "none" || !strings.Contains(rc.Note, "no lab minted") { + t.Fatalf("answer %+v, receipt %+v, err %v", a, rc, err) + } +} + +// Criterion 3: turned on, the adapter decides and the receipt names the route. +func TestOnTheAdapterDecidesAndTheReceiptNamesTheRoute(t *testing.T) { + adapter, rec := &fake{value: "iss-2", p: 0.93}, &pairs{} + host := &fake{value: "iss-1", p: 1} + a, rc, err := router(oracle.JudgementOn, adapter, rec).Decide(context.Background(), State{Text: "s"}, dupQuestion(), host) + if err != nil { + t.Fatal(err) + } + if a.Value != "iss-2" || a.Probability != 0.93 || host.calls != 0 { + t.Fatalf("answer %+v, host calls %d: the adapter decides", a, host.calls) + } + if rc.Route != RouteAdapter || rc.Target != "openrouter/typesafe/jev-1.13" || rc.ModelReported != "typesafe/jev-1.13-0901" || + rc.Fallback != "" || len(rec.got) != 0 { + t.Fatalf("receipt %+v, pairs %d", rc, len(rec.got)) + } +} + +// Criterion 3: an unreachable adapter falls back to the host with a receipt +// saying so. +func TestOnAnUnreachableAdapterFallsBackToTheHostWithAReceipt(t *testing.T) { + unreachable := &fake{err: openaiapi.ErrUnreachable} + host := &fake{value: "iss-1", p: 1} + a, rc, err := router(oracle.JudgementOn, unreachable, &pairs{}).Decide(context.Background(), State{Text: "s"}, dupQuestion(), host) + if err != nil || a.Value != "iss-1" || host.calls != 1 { + t.Fatalf("answer %+v, host calls %d, err %v", a, host.calls, err) + } + if rc.Route != RouteHost || !strings.Contains(rc.Fallback, "openrouter/typesafe/jev-1.13") || !strings.Contains(rc.Fallback, "the host decided") { + t.Fatalf("receipt = %+v", rc) + } + // A host asked out of band (the drain's request file) is pending: the + // fallback is still in the receipt. + _, rc, err = router(oracle.JudgementOn, unreachable, &pairs{}).Decide(context.Background(), State{Text: "s"}, dupQuestion(), Pending()) + if !errors.Is(err, ErrHostPending) || rc.Fallback == "" { + t.Fatalf("pending host: receipt %+v, err %v", rc, err) + } + // A state over the window is a fallback too, never a call. + rt := router(oracle.JudgementOn, unreachable, &pairs{}) + rt.Window = 10 + unreachable.calls = 0 + _, rc, _ = rt.Decide(context.Background(), State{Text: strings.Repeat("x", 400)}, dupQuestion(), host) + if unreachable.calls != 0 || !strings.Contains(rc.Fallback, "window") { + t.Fatalf("over-window on: calls %d, receipt %+v", unreachable.calls, rc) + } +} + +// Criterion 6: one interface, so a type is switched by configuration alone. +// The same caller, the same arguments: host, shadow, on. +func TestOneInterfaceSwitchesATypeWithoutTheCallerChanging(t *testing.T) { + caller := func(rt *Router) (Answer, Receipt) { + a, rc, err := rt.Decide(context.Background(), State{Text: "s"}, dupQuestion(), Answered("iss-1")) + if err != nil { + t.Fatal(err) + } + return a, rc + } + for _, c := range []struct { + mode, value, route string + pairs int + }{ + {oracle.JudgementHost, "iss-1", RouteHost, 0}, + {oracle.JudgementShadow, "iss-1", RouteHost, 1}, + {oracle.JudgementOn, "iss-2", RouteAdapter, 0}, + } { + adapter, rec := &fake{value: "iss-2", p: 0.9}, &pairs{} + a, rc := caller(router(c.mode, adapter, rec)) + if a.Value != c.value || rc.Route != c.route || len(rec.got) != c.pairs || rc.Mode != c.mode { + t.Errorf("%s: answer %+v receipt %+v pairs %d", c.mode, a, rc, len(rec.got)) + } + if c.mode == oracle.JudgementHost && adapter.calls != 0 { + t.Errorf("host: the adapter was called") + } + } + // Both implementations satisfy the one interface. + var _ Judge = Host{} + var _ Judge = Adapter{} +} + +// The registry is closed: a type outside it, or a question that does not hold +// together, is refused before anything is asked. +func TestTheRegistryIsClosedAndAQuestionIsChecked(t *testing.T) { + rt := router(oracle.JudgementOn, &fake{value: "x"}, &pairs{}) + host := &fake{value: "none"} + if _, _, err := rt.Decide(context.Background(), State{Text: "s"}, Question{Type: "lesson-ranking", Kind: KindChoose, Ask: "?", Options: []string{"a"}}, host); err == nil { + t.Fatal("a type outside the registry was asked") + } + if _, _, err := rt.Decide(context.Background(), State{Text: "s"}, Question{Type: DuplicateMatch, Kind: KindChoose, Ask: "?"}, host); err == nil { + t.Fatal("a choose question with no options was asked") + } + if host.calls != 0 { + t.Fatal("the host was asked a refused question") + } + for _, ty := range Types() { + if !Known(string(ty)) { + t.Errorf("%s is listed but not known", ty) + } + } + if Known("embark-lesson-ranking") { + t.Error("embark's lesson ranking is a term-overlap computation, not a judgement (iss-2610262033055327)") + } +} + +// An adapter's answer outside the question's options is not an answer: turned +// on, the host decides instead, with a receipt. +func TestOnAnAnswerOutsideTheOptionsFallsBack(t *testing.T) { + host := &fake{value: "none"} + a, rc, err := router(oracle.JudgementOn, &fake{value: "iss-9", p: 0.99}, &pairs{}).Decide(context.Background(), State{Text: "s"}, dupQuestion(), host) + if err != nil || a.Value != "none" || !strings.Contains(rc.Fallback, "not one of") { + t.Fatalf("answer %+v receipt %+v err %v", a, rc, err) + } +} + +func TestTheHostParsesItsAnswerAtProbabilityOne(t *testing.T) { + for _, c := range []struct { + q Question + text string + want string + bad bool + }{ + {dupQuestion(), " ISS-2 \n", "iss-2", false}, + {dupQuestion(), "iss-7", "", true}, + {Question{Type: RemedyRead, Kind: KindYesNo, Ask: "Does it change what a user sees?"}, "Yes", "yes", false}, + {Question{Type: LedgerSeverity, Kind: KindScore, Ask: "How severe?", Min: 1, Max: 5}, "4", "4", false}, + {Question{Type: LedgerSeverity, Kind: KindScore, Ask: "How severe?", Min: 1, Max: 5}, "9", "", true}, + } { + a, err := Answered(c.text).Judge(context.Background(), State{Text: "s"}, c.q) + if c.bad { + if err == nil { + t.Errorf("%q: parsed as %+v, want a refusal", c.text, a) + } + continue + } + if err != nil || a.Value != c.want || a.Probability != 1 { + t.Errorf("%q: %+v, %v", c.text, a, err) + } + } + // The host is asked the prompt the adapter is asked: like for like. + var asked string + h := Host{Ask: func(_ context.Context, prompt string) (string, error) { asked = prompt; return "none", nil }} + if _, err := h.Judge(context.Background(), State{Text: "the state"}, dupQuestion()); err != nil { + t.Fatal(err) + } + if asked != Render(State{Text: "the state"}, dupQuestion()) || !strings.Contains(asked, "the state") || !strings.Contains(asked, "iss-2") { + t.Fatalf("host prompt = %q", asked) + } +} + +// caller is a fake oracle call: it checks the request and answers content. +type caller struct { + content string + req oracle.CallRequest +} + +func (c *caller) Call(_ context.Context, _ credential.Source, req oracle.CallRequest, _ ...openaiapi.Option) ([]byte, oracle.CallRecord, error) { + c.req = req + if req.Contract != nil { + if err := req.Contract([]byte(c.content)); err != nil { + return nil, oracle.CallRecord{}, err + } + } + return []byte(c.content), oracle.CallRecord{Provider: req.Target.Provider, ModelAsked: req.Target.Model, ModelReported: "typesafe/jev-1.13-0901"}, nil +} + +func TestTheAPIAdapterParsesTheTypedAnswer(t *testing.T) { + c := &caller{content: "```json\n{\"value\": \"iss-2\", \"probability\": 0.87}\n```"} + a, err := Adapter{Calls: c, Target: target}.Judge(context.Background(), State{Text: "s"}, dupQuestion()) + if err != nil || a.Value != "iss-2" || a.Probability != 0.87 || a.Model != "typesafe/jev-1.13-0901" { + t.Fatalf("answer %+v, err %v", a, err) + } + if c.req.Target != target || !strings.Contains(c.req.Brief.Input, "iss-1") || !strings.Contains(c.req.Brief.Instructions, "probability") { + t.Fatalf("request = %+v", c.req) + } + for _, bad := range []string{`{"value":"iss-2","probability":1.5}`, `{"value":"iss-9","probability":0.5}`, `not json`, `{"value":"iss-2"}`} { + c.content = bad + if a, err := (Adapter{Calls: c, Target: target}).Judge(context.Background(), State{Text: "s"}, dupQuestion()); err == nil { + t.Errorf("%s: parsed as %+v", bad, a) + } + } +} + +// Criterion 4: a configured model not on its provider's list is refused when +// the configuration is read, before any call; a listed one loads with its mode. +func TestLoadRefusesAnUnlistedModelBeforeAnyCall(t *testing.T) { + roots := layered.Roots{Repo: t.TempDir(), Home: t.TempDir()} + write := func(body string) { + p := abcdhome.Path(roots.Home, "config.json") + if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + block := `"api":{"openrouter":{"base_url":"https://openrouter.ai/api/v1","key":"openrouter","models":["typesafe/jev-1.13"]}}` + write(`{"oracle":{` + block + `,"judgements":{"duplicate-match":"shadow:openrouter/mistral/small"}}}`) + if _, err := Load(roots, roots.Repo); err == nil || !strings.Contains(err.Error(), "refused before any call") { + t.Fatalf("Load = %v, want the allowlist refusal", err) + } + write(`{"oracle":{` + block + `,"judgements":{"duplicate-match":"shadow:openrouter/typesafe/jev-1.13", + "remedy-read":"openrouter/typesafe/jev-1.13","lesson-ranking":"openrouter/typesafe/jev-1.13"}}}`) + rt, err := Load(roots, roots.Repo) + if err != nil { + t.Fatal(err) + } + if rt.Mode(DuplicateMatch) != oracle.JudgementShadow || rt.Mode(RemedyRead) != oracle.JudgementOn || rt.Mode(MemoryCheck) != oracle.JudgementHost { + t.Fatalf("modes: %s %s %s", rt.Mode(DuplicateMatch), rt.Mode(RemedyRead), rt.Mode(MemoryCheck)) + } + if len(rt.Diagnostics) != 1 || !strings.Contains(rt.Diagnostics[0], "lesson-ranking") { + t.Fatalf("diagnostics = %q, want the unknown type named", rt.Diagnostics) + } +} diff --git a/internal/core/decide/judge/router.go b/internal/core/decide/judge/router.go new file mode 100644 index 0000000000..da2b0669c2 --- /dev/null +++ b/internal/core/decide/judge/router.go @@ -0,0 +1,264 @@ +package judge + +// router.go is the per-type switch (spc-2609221011151661 scopes 3, 5 and 6). +// Each judgement type is routed as oracle.judgements. says, read and +// checked against the provider's allowlist when the configuration is loaded, +// before any call (adr-2609221009491186, criterion 4): +// +// - host, or nothing configured: the host decides, and nothing else runs. +// - shadow:/: the host decides; the adapter answers beside +// it and decides nothing; the pair is recorded in the newest lab home +// (criterion 1). A state larger than the model's window is not sent: the +// pair is recorded as skipped with its size (criterion 5). +// - /: the adapter decides and the receipt names the +// route; an adapter that cannot answer — unreachable, failed, an answer +// outside the question's choices, a state over its window — leaves the +// host to decide, with a receipt saying why (criterion 3). +// +// The caller hands Decide its host judgement and nothing else changes when a +// type is switched (criterion 6). + +import ( + "context" + "errors" + "fmt" + "math" + "sort" + "strings" + + "github.com/intentdriven/abcd/internal/core/credential" + "github.com/intentdriven/abcd/internal/core/lab" + "github.com/intentdriven/abcd/internal/core/layered" + "github.com/intentdriven/abcd/internal/core/oracle" + "github.com/intentdriven/abcd/internal/termsafe" +) + +// DefaultWindow is the decision model's context, in tokens: the first +// candidate's 32K (itd-2609221009495079 scope condition 1). +const DefaultWindow = 32 << 10 + +// The routes a receipt names. +const ( + RouteHost = "host" + RouteAdapter = "adapter" +) + +// maxDetail bounds an error's account in a pair or a receipt. +const maxDetail = 300 + +// Receipt is what a judgement's run record carries: the type and its mode, the +// route that decided, the adapter's target and the model it reported, the +// probability the deciding side gave, the state's hash and size, why a type +// turned on fell back to the host, and where a shadow pair went or why it was +// not kept. +type Receipt struct { + Type Type `json:"type"` + Mode string `json:"mode"` + Route string `json:"route"` + Target string `json:"target,omitempty"` + ModelReported string `json:"model_reported,omitempty"` + Probability float64 `json:"probability"` + StateSHA256 string `json:"state_sha256"` + StateTokens int `json:"state_tokens"` + Fallback string `json:"fallback,omitempty"` + Pair string `json:"pair,omitempty"` + PairSkipped string `json:"pair_skipped,omitempty"` + Note string `json:"note,omitempty"` +} + +// Recorder keeps a shadow pair and returns the lab and the file it went to. +type Recorder func(lab.Pair) (labID, where string, err error) + +// Router routes each judgement type to the host, to the host with the adapter +// in shadow, or to the adapter. +type Router struct { + routes map[Type]oracle.Target + adapter func(oracle.Target) Judge + record Recorder + // Window is the adapter's window in tokens; DefaultWindow when zero. + Window int + // Diagnostics are the configuration's non-fatal reports, for a front + // door to print: the oracle reader's, and a route naming a type outside + // the registry. + Diagnostics []string +} + +// New is a Router over routes (a type absent from it stays on the host), the +// adapter each target is judged by, and where shadow pairs are recorded. +func New(routes map[Type]oracle.Target, adapter func(oracle.Target) Judge, record Recorder) *Router { + r := &Router{routes: map[Type]oracle.Target{}, adapter: adapter, record: record} + for t, tgt := range routes { + r.routes[t] = tgt + } + return r +} + +// Load reads oracle.judgements. through the oracle reader, which refuses +// a model not on its provider's allowlist (or one oracle.denylist matches) +// before any call, and returns the Router the repository at repoRoot judges +// with: the adapter is the API adapter with the key resolved by name from the +// account's credential store, and shadow pairs go to the newest lab minted for +// the repository. A route to a type outside the registry is a diagnostic. +func Load(roots layered.Roots, repoRoot string) (*Router, error) { + cfg, err := oracle.LoadAPI(roots) + if err != nil { + return nil, err + } + routes := map[Type]oracle.Target{} + var diags []string + diags = append(diags, cfg.Diagnostics...) + for _, pr := range cfg.Routes() { + if pr.Kind != "judgement" { + continue + } + if !Known(pr.Name) { + diags = append(diags, fmt.Sprintf("decision adapter: oracle.judgements.%s in %s is not a judgement type (%s); it is skipped", + termsafe.Sanitize(clip(pr.Name)), termsafe.Sanitize(pr.Target.Origin), typeNames())) + continue + } + routes[Type(pr.Name)] = pr.Target + } + creds := credential.Store(roots.Home) + r := New(routes, func(t oracle.Target) Judge { return Adapter{Calls: cfg, Creds: creds, Target: t} }, + func(p lab.Pair) (string, string, error) { return lab.RecordPair(repoRoot, p) }) + r.Diagnostics = diags + return r, nil +} + +// Mode is how t is routed: oracle.JudgementHost, JudgementShadow or +// JudgementOn. +func (r *Router) Mode(t Type) string { + if tgt, ok := r.routes[t]; ok && (tgt.Mode == oracle.JudgementShadow || tgt.Mode == oracle.JudgementOn) { + return tgt.Mode + } + return oracle.JudgementHost +} + +func (r *Router) window() int { + if r.Window > 0 { + return r.Window + } + return DefaultWindow +} + +// Decide answers q over s on the route t's configuration names, with host as +// the host's judgement. Its error is the host's (ErrHostPending among them) +// or the question's refusal; an adapter's failure is never an error: in shadow +// it is recorded in the pair, and turned on it is a fallback to host. +func (r *Router) Decide(ctx context.Context, s State, q Question, host Judge) (Answer, Receipt, error) { + if err := q.Check(); err != nil { + return Answer{}, Receipt{}, err + } + mode := r.Mode(q.Type) + size := tokens(Render(s, q)) + rc := Receipt{Type: q.Type, Mode: mode, Route: RouteHost, StateSHA256: s.SHA256(), StateTokens: size} + tgt := r.routes[q.Type] + if mode != oracle.JudgementHost { + rc.Target = tgt.String() + } + switch mode { + case oracle.JudgementOn: + if size > r.window() { + rc.Fallback = fmt.Sprintf("the state is about %d tokens, more than %s's window of %d, so it was not sent and the host decided", + size, rc.Target, r.window()) + return r.byHost(ctx, s, q, host, rc) + } + a, err := r.ask(ctx, tgt, s, q) + if err != nil { + rc.Fallback = fmt.Sprintf("%s did not answer (%s), so the host decided", rc.Target, detail(err)) + return r.byHost(ctx, s, q, host, rc) + } + rc.Route, rc.ModelReported, rc.Probability = RouteAdapter, a.Model, a.Probability + return a, rc, nil + case oracle.JudgementShadow: + a, rc, err := r.byHost(ctx, s, q, host, rc) + if err != nil { + return a, rc, err + } + pair := lab.Pair{Type: string(q.Type), Kind: string(q.Kind), StateSHA256: rc.StateSHA256, StateTokens: size, + Host: a.Value, Target: rc.Target} + if size > r.window() { + pair.Skipped = lab.SkippedWindow + pair.Detail = fmt.Sprintf("the state is about %d tokens, more than the window of %d; the host decided alone", size, r.window()) + } else if sa, err := r.ask(ctx, tgt, s, q); err != nil { + pair.Skipped, pair.Detail = lab.SkippedFailed, detail(err) + } else { + p := sa.Probability + pair.Adapter, pair.Probability, pair.ModelReported = sa.Value, &p, sa.Model + } + rc.PairSkipped = pair.Skipped + if r.record == nil { + rc.Note = "the shadow pair was not kept: no recorder" + return a, rc, nil + } + if _, where, err := r.record(pair); err != nil { + rc.Note = "the shadow pair was not kept: " + detail(err) + } else { + rc.Pair = where + } + return a, rc, nil + default: + return r.byHost(ctx, s, q, host, rc) + } +} + +// byHost is the host's decision, on rc. +func (r *Router) byHost(ctx context.Context, s State, q Question, host Judge, rc Receipt) (Answer, Receipt, error) { + if host == nil { + return Answer{}, rc, ErrHostPending + } + a, err := host.Judge(ctx, s, q) + if err != nil { + return Answer{}, rc, err + } + rc.Route, rc.Probability = RouteHost, a.Probability + return a, rc, nil +} + +// ask is the adapter's answer, held to the question: a value among its +// choices and a probability from 0 to 1, whatever implementation answered. +func (r *Router) ask(ctx context.Context, tgt oracle.Target, s State, q Question) (Answer, error) { + if r.adapter == nil { + return Answer{}, errors.New("no adapter") + } + j := r.adapter(tgt) + if j == nil { + return Answer{}, errors.New("no adapter") + } + a, err := j.Judge(ctx, s, q) + if err != nil { + return Answer{}, err + } + v, ok := q.match(a.Value) + if !ok { + return Answer{}, fmt.Errorf("its answer %q is not one of %s", termsafe.Sanitize(clip(a.Value)), strings.Join(q.Choices(), ", ")) + } + if math.IsNaN(a.Probability) || a.Probability < 0 || a.Probability > 1 { + return Answer{}, fmt.Errorf("its probability %v is not from 0 to 1", a.Probability) + } + a.Value = v + return a, nil +} + +// Routes are the types routed off the host, sorted, with their routes as the +// configuration spells them, for a board. +func (r *Router) Routes() []string { + var out []string + for t, tgt := range r.routes { + if m := r.Mode(t); m != oracle.JudgementHost { + out = append(out, string(t)+" = "+tgt.Route()) + } + } + sort.Strings(out) + return out +} + +// detail is an error's account for a pair or a receipt: terminal-safe, one +// line, bounded. +func detail(err error) string { + s := strings.Join(strings.Fields(termsafe.Sanitize(err.Error())), " ") + if len(s) > maxDetail { + s = s[:maxDetail] + "…" + } + return s +} From 966013a1198df9743fa14b16462924af331ddf0b Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:31:49 +0000 Subject: [PATCH 05/20] docs: drop embark's lesson ranking from the decision adapter's judgement types The embark page ranks retrospective lessons by term overlap against the new voyage's brief, a deterministic computation with no host answer to pair, so it cannot run in shadow or be turned over. The intent's two lists no longer name it and the judge registry never held it. Recorded as facilitator-on-delegation: a factual correction to the intent's scope, accepted by the run's coordinator. Resolves: iss-2610262033055327 Assisted-by: Claude:claude-opus-5-5 --- ...09221009495079-abcd-s-typed-judgements-can-run-through.md | 4 ++-- ...ss-2610262033055327-the-decision-adapter-intent-treats.md | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) rename .abcd/work/issues/{open => resolved}/iss-2610262033055327-the-decision-adapter-intent-treats.md (86%) diff --git a/.abcd/development/intents/planned/itd-2609221009495079-abcd-s-typed-judgements-can-run-through.md b/.abcd/development/intents/planned/itd-2609221009495079-abcd-s-typed-judgements-can-run-through.md index d92a6a439b..d18990bea1 100644 --- a/.abcd/development/intents/planned/itd-2609221009495079-abcd-s-typed-judgements-can-run-through.md +++ b/.abcd/development/intents/planned/itd-2609221009495079-abcd-s-typed-judgements-can-run-through.md @@ -25,7 +25,7 @@ related_adrs: [adr-2609221009491186] ## Why This Matters -The research pass of 2026-09-22 found Jev (TypeSafe AI) to be a typed-decision model, not a text model and not a router: choose one of N, score against a rubric, yes or no, each with a calibrated probability, in under a second at $0.042 per million tokens, 32K context, and unable to abstain. Independent evidence is one forty-case test; the vendor's numbers are vendor-graded. abcd's judgement-shaped steps are many: the remedy read, duplicate matching, unmeetable criteria, the eligibility residual, source classification and selection in the library, memory checks, ledger category and severity, the pre-pass's overlap, the docs lint's present-tense advisory, embark's lesson ranking. Ruled: an adapter behind one interface, the host as default, shadow mode as an abcd lab, a research note as the evidence a ruling turns a judgement type on with. +The research pass of 2026-09-22 found Jev (TypeSafe AI) to be a typed-decision model, not a text model and not a router: choose one of N, score against a rubric, yes or no, each with a calibrated probability, in under a second at $0.042 per million tokens, 32K context, and unable to abstain. Independent evidence is one forty-case test; the vendor's numbers are vendor-graded. abcd's judgement-shaped steps are many: the remedy read, duplicate matching, unmeetable criteria, the eligibility residual, source classification and selection in the library, memory checks, ledger category and severity, the pre-pass's overlap, the docs lint's present-tense advisory. Ruled: an adapter behind one interface, the host as default, shadow mode as an abcd lab, a research note as the evidence a ruling turns a judgement type on with. ## Mechanism @@ -40,7 +40,7 @@ We expect calibrated typed answers to agree with the host on closed-option judge ## What's In Scope - **One interface**: `(state, typed question) → (value, probability)` with three question kinds (choose, score, yes/no); the host's own judgement implements it too and stays the default. -- **The judgement types** it may be configured for: the remedy read, duplicate matching, unmeetable criteria, the eligibility residual, source classification and selection over a shortlist, the memory checks, ledger category and severity, the pre-pass's overlap, the docs lint's present-tense advisory, embark's lesson ranking. +- **The judgement types** it may be configured for: the remedy read, duplicate matching, unmeetable criteria, the eligibility residual, source classification and selection over a shortlist, the memory checks, ledger category and severity, the pre-pass's overlap, the docs lint's present-tense advisory. Embark's lesson ranking is not one: the embark page ranks lessons by term overlap against the new voyage's brief, a computation with no host answer to pair (iss-2610262033055327). - **Shadow mode as a lab**: `abcd lab mint` opens a lab home; the adapter runs beside the host on the configured types, decides nothing, and each pair (host's answer, adapter's answer, probability, the state's hash) is recorded under the lab home in the machine-scoped store, never the repository. - **The evidence**: `abcd lab harvest` produces agreement rates per type with counts and probability bands; the research note filed from it (dated, under the record's research notes) is the evidence a ruling turns a type on with. - **Turning on**: per judgement type in configuration; unreachable falls back to the host with a receipt; the run record names which route judged. diff --git a/.abcd/work/issues/open/iss-2610262033055327-the-decision-adapter-intent-treats.md b/.abcd/work/issues/resolved/iss-2610262033055327-the-decision-adapter-intent-treats.md similarity index 86% rename from .abcd/work/issues/open/iss-2610262033055327-the-decision-adapter-intent-treats.md rename to .abcd/work/issues/resolved/iss-2610262033055327-the-decision-adapter-intent-treats.md index 06a7808c38..6e76d9b82b 100644 --- a/.abcd/work/issues/open/iss-2610262033055327-the-decision-adapter-intent-treats.md +++ b/.abcd/work/issues/resolved/iss-2610262033055327-the-decision-adapter-intent-treats.md @@ -12,6 +12,11 @@ found_at: ".abcd/development/intents/planned/itd-2609221009495079-abcd-s-typed-j remedy: "In the decision-adapter intent drop embark's lesson ranking from the judgement-type lists (press-release rationale and scope bullet), or add a host judgement for it first: the shipped ranking is a deterministic term-overlap computation with no host answer to pair." related_intents: [itd-2609221009495079] finding_ref: cfk-0fa68fc1ec4b7f88 +resolution: "embark's lesson ranking is dropped from the decision adapter intent's judgement types and is not in the judge package's registry: the embark page ranks lessons by term overlap, a computation with no host answer to pair" +impact: internal +resolved_by: + intent: "itd-2609221009495079" + spec: "spc-2609221011151661" --- The decision-adapter intent treats embark's lesson ranking as a judgement the host answers by default; the embark page ranks lessons by term overlap, with no judgement in that step. From 7b30d0e5c501fa274bfb7baf88439e61226e4a32 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:32:32 +0000 Subject: [PATCH 06/20] docs: describe the decision adapter's routes, shadow pairs and harvest agreement The configuration chapter gives oracle.judgements' three modes, the judgement type registry and the drain's eligibility-residual routes; the adapters chapter gains the decision adapter; the lab command page says what the harvest's agreement carries and that turning a type on is the user's decision. Assisted-by: Claude:claude-opus-5-5 --- .../brief/05-internals/02-adapters.md | 33 +++++++++++++++++ .../brief/05-internals/03-configuration.md | 37 ++++++++++++++++++- commands/lab.md | 14 +++++++ 3 files changed, 83 insertions(+), 1 deletion(-) diff --git a/.abcd/development/brief/05-internals/02-adapters.md b/.abcd/development/brief/05-internals/02-adapters.md index 4c0ab7a55e..16bf4c6227 100644 --- a/.abcd/development/brief/05-internals/02-adapters.md +++ b/.abcd/development/brief/05-internals/02-adapters.md @@ -247,6 +247,39 @@ is sent to a provider. No test reaches a real provider: the client and every dispatching verb are exercised end to end against a fake on the loopback address, and the key is built at run time. +### The decision adapter — typed judgements, measured in a lab first + +The decision adapter (itd-2609221009495079, `core/decide/judge`) answers abcd's +closed-option judgements with a value and a probability. It has no generative +step. Every judgement goes through one interface, `(state, typed question) → +(value, probability)`. A question is one of three kinds: choose one of N, +score on a scale, or yes/no. Its type is one of a closed registry +([`03-configuration.md`](03-configuration.md) lists it). The host's own +judgement implements the interface at probability 1, and so does the decision +model reached through the API adapter above, so the two are compared like for +like. The adapter renders the same prompt the host is shown, sends it through +`APIConfig.Call` (which admits the target against the allowlist and the +denylist again and resolves the key by name), and reads back one JSON object +with a `value` among the question's choices and a `probability` from 0 to 1. +The model cannot abstain, so any other answer is not an answer. + +A router switches each type by configuration alone, so no caller changes when +a type moves. With `host`, or nothing set, the host decides. With +`shadow:/`, the host decides and the adapter's answer is +recorded beside it, as a pair in the newest lab home's `pairs.jsonl`, never in +the repository. A state larger than the model's 32K window is recorded as +skipped with its size. With a bare `/`, the adapter decides, +and the run record names the route and the model the provider reported. An +adapter that cannot answer leaves the decision to the host, with a fallback +receipt saying why. The allowlist refusal of adr-2609221009491186 happens when +the configuration is read, before any call. `abcd lab harvest` turns the pairs +into agreement per type, with counts and probability bands, and the research +note filed from it is the evidence a ruling turns a type on with. The first +candidate is `typesafe/jev-1.13` on OpenRouter's list. The live paid check +against it is owed to a person (iss-2609291925398978). The tests drive the +router, the pairs file, the harvest and the drain's judgement with a fake +adapter and a temporary home. + ### Reviewer routes — the command-line runner, and RepoPrompt over MCP A build lane's ruthless and security reviews, which the validate stage hands diff --git a/.abcd/development/brief/05-internals/03-configuration.md b/.abcd/development/brief/05-internals/03-configuration.md index aa7b383581..8f8bcd9856 100644 --- a/.abcd/development/brief/05-internals/03-configuration.md +++ b/.abcd/development/brief/05-internals/03-configuration.md @@ -210,7 +210,8 @@ rather than skipped: }, "denylist": ["openai/*"], // optional, none bundled; repo or machine "roles": { "scribe": "openrouter/typesafe/jev-1.13" }, // an agent in the roster; a keyed - "judgements": { "duplicate-match": "openrouter/typesafe/jev-1.13" } // provider's routes: machine only + "judgements": { "duplicate-match": "shadow:openrouter/typesafe/jev-1.13" } // provider's routes: machine only; + // host | shadow:/ | / } } ``` @@ -335,6 +336,40 @@ rather than skipped: one, applies in its place (ruling CD2 of 2026-09-29). The same fault in `~/.abcd.noindex/config.json` is refused, because that file is the person's own and a route they set is never dropped silently. +- **A judgement type's route has a mode** (itd-2609221009495079). An + `oracle.judgements.` value is `host`, `shadow:/` or + `/`. `host`, or no route, keeps the type on the host. A + repository's `host` holds against the machine's route to the same name, + since it only keeps work on the host. In **shadow** the host still decides. + The decision adapter is asked the same typed question over the same state + beside it and decides nothing. Each pair (the host's answer, the adapter's + answer and probability, the state's SHA-256 and size) is appended to + `pairs.jsonl` in the newest lab home the repository's lane of + `~/.abcd.noindex/lab/` holds, so `abcd lab mint` comes first. A state larger + than the model's window (32K tokens, counted high at three bytes a token) is + never sent: its pair is recorded as skipped with the size. With no lab + minted, the host still decides and no pair is kept. A bare + `/` **turns the type on**: the adapter decides, and the run + record names the route and the model that answered. If the adapter cannot + answer (unreachable, failed, an answer outside the question's choices, a + state over its window), the host decides instead, with a receipt saying + why. A shadow route passes every rule a route turned on does (the + allowlist, the denylist, the keyed-provider skip, the malformed-route skip) + when the configuration is read and before any call. A role takes no mode. + The types are a closed registry: `remedy-read`, `duplicate-match`, + `unmeetable-criteria`, `eligibility-residual`, `source-classification`, + `source-selection`, `memory-check`, `ledger-category`, `ledger-severity`, + `prepass-overlap`, `present-tense-advisory`. A route naming anything else is + a diagnostic and is skipped. The drain's judgement before a lane opens is + `eligibility-residual`, a choice of `no`, `user-visible` or `trust-rule`. In + shadow its pair is recorded once the host's or the routed judge's answer is + taken. Turned on, the adapter is asked once the host's request is written, + and its answer is taken the way a routed judge's is, with the judgement's + route naming `decision-adapter /`. An adapter that cannot + answer leaves the host's request standing, with a fallback receipt beside + the routed judge's. `abcd lab harvest` reports the agreement per type, and + the research note filed from it is the evidence a ruling turns a type on + with. - **A route to a provider that holds a key sits on the machine alone.** Only a route the person set up on their own machine may spend their paid key (the product thinker's ruling AA(b) of 2026-09-29), so a repository's diff --git a/commands/lab.md b/commands/lab.md index 7812738903..9a240fdfc7 100644 --- a/commands/lab.md +++ b/commands/lab.md @@ -96,6 +96,20 @@ yourself**: show the candidates and let the user decide, and note any without a `refutation`. When a finding cites no probe record or evidence, or one that is missing or incomplete, `gaps` names it, the exit is 1 and nothing is written. +When the lab collected the decision adapter's shadow pairs, the harvest also +carries `agreement`. A judgement type is put in shadow by setting +`oracle.judgements.` to `shadow:/` in +`~/.abcd.noindex/config.json`. The host keeps deciding, and each answer the +adapter gives beside it is appended to `pairs.jsonl` in the newest lab home. +For each type, `agreement` gives `pairs`, `compared` (the adapter answered), +`agreed`, `skipped` (the state was over the model's window, or the call +failed), `rate`, and the compared and agreed counts in four probability +`bands`. It ends with the `note` line, naming the lab home and the count. The +drain's judgement before a lane opens is the type `eligibility-residual`. Show +the user the rates and the note: a research note filed from it is the evidence +a ruling turns a type on with (`/`, no `shadow:`), and turning +a type on is the user's decision, never yours. + **Plan a model matrix run** in a lab whose preflight passed: ```bash From 443f0f3a837695a840d8efdec01b2656d6d8fa95 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:32:58 +0000 Subject: [PATCH 07/20] docs: say the drain's decision routing is built but not yet called No judgement step calls the router yet: the drain's call is a later change and the other types' judgements are made in command pages. The configuration and adapters chapters now say so instead of describing the drain's routes as running. Assisted-by: Claude:claude-opus-5-5 --- .../brief/05-internals/02-adapters.md | 7 +++++-- .../brief/05-internals/03-configuration.md | 19 ++++++++++++------- commands/lab.md | 3 +-- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/.abcd/development/brief/05-internals/02-adapters.md b/.abcd/development/brief/05-internals/02-adapters.md index 16bf4c6227..3bc831b796 100644 --- a/.abcd/development/brief/05-internals/02-adapters.md +++ b/.abcd/development/brief/05-internals/02-adapters.md @@ -276,8 +276,11 @@ the configuration is read, before any call. `abcd lab harvest` turns the pairs into agreement per type, with counts and probability bands, and the research note filed from it is the evidence a ruling turns a type on with. The first candidate is `typesafe/jev-1.13` on OpenRouter's list. The live paid check -against it is owed to a person (iss-2609291925398978). The tests drive the -router, the pairs file, the harvest and the drain's judgement with a fake +against it is owed to a person (iss-2609291925398978). The drain's judgement +(`eligibility-residual`) has its routing built but is not yet called from the +drain's move, and no other type's step calls the router yet, so until those +calls are added every judgement stays on the host. The tests drive the +router, the pairs file, the harvest and the drain's routing with a fake adapter and a temporary home. ### Reviewer routes — the command-line runner, and RepoPrompt over MCP diff --git a/.abcd/development/brief/05-internals/03-configuration.md b/.abcd/development/brief/05-internals/03-configuration.md index 8f8bcd9856..1363bca41c 100644 --- a/.abcd/development/brief/05-internals/03-configuration.md +++ b/.abcd/development/brief/05-internals/03-configuration.md @@ -361,13 +361,18 @@ rather than skipped: `source-selection`, `memory-check`, `ledger-category`, `ledger-severity`, `prepass-overlap`, `present-tense-advisory`. A route naming anything else is a diagnostic and is skipped. The drain's judgement before a lane opens is - `eligibility-residual`, a choice of `no`, `user-visible` or `trust-rule`. In - shadow its pair is recorded once the host's or the routed judge's answer is - taken. Turned on, the adapter is asked once the host's request is written, - and its answer is taken the way a routed judge's is, with the judgement's - route naming `decision-adapter /`. An adapter that cannot - answer leaves the host's request standing, with a fallback receipt beside - the routed judge's. `abcd lab harvest` reports the agreement per type, and + meant to run as `eligibility-residual`, a choice of `no`, `user-visible` or + `trust-rule`. Its routing is built (`loop/drain_decide.go`). In shadow, the + routing records a pair once the host's or the routed judge's answer is + taken. Turned on, it asks the adapter once the host's request is written, + and takes the answer the way a routed judge's is taken, with the route named + `decision-adapter /`. If the adapter cannot answer, the + host's request stands, with a fallback receipt beside the routed judge's. + Nothing in the drain's move calls this routing yet, so until that call is + added the drain's judgement stays on the host whatever this route says. No + other type's step calls the router yet either. Those judgements are made in + the host's command pages, which hand nothing back to the binary, so a route + to one of them takes effect only once its step calls the router. `abcd lab harvest` reports the agreement per type, and the research note filed from it is the evidence a ruling turns a type on with. - **A route to a provider that holds a key sits on the machine alone.** Only a diff --git a/commands/lab.md b/commands/lab.md index 9a240fdfc7..42a206d17c 100644 --- a/commands/lab.md +++ b/commands/lab.md @@ -104,8 +104,7 @@ adapter gives beside it is appended to `pairs.jsonl` in the newest lab home. For each type, `agreement` gives `pairs`, `compared` (the adapter answered), `agreed`, `skipped` (the state was over the model's window, or the call failed), `rate`, and the compared and agreed counts in four probability -`bands`. It ends with the `note` line, naming the lab home and the count. The -drain's judgement before a lane opens is the type `eligibility-residual`. Show +`bands`. It ends with the `note` line, naming the lab home and the count. Show the user the rates and the note: a research note filed from it is the evidence a ruling turns a type on with (`/`, no `shadow:`), and turning a type on is the user's decision, never yours. From 26f0e61936eaa4a5c1afad9e8f1351c66c680075 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 29 Oct 2026 06:00:58 +0000 Subject: [PATCH 08/20] feat: route the drain's judgement through the decision adapter The drain's judgement over a remedy is the typed judgement eligibility-residual: no, user-visible or trust-rule. shadowDrainJudgement records the adapter's answer beside a taken judgement when the type is in shadow; decideDrainJudgement asks the adapter when it is turned on, takes its answer as a routed judge's under the same disposition rules with the route naming the adapter and model, and returns a fallback receipt, the host's request standing, when it cannot answer. The call from the drain's move is left for a change that holds drain.go. Assisted-by: Claude:claude-opus-5-5 --- internal/core/implement/loop/drain_decide.go | 166 +++++++++++++++++ .../core/implement/loop/drain_decide_test.go | 174 ++++++++++++++++++ 2 files changed, 340 insertions(+) create mode 100644 internal/core/implement/loop/drain_decide.go create mode 100644 internal/core/implement/loop/drain_decide_test.go diff --git a/internal/core/implement/loop/drain_decide.go b/internal/core/implement/loop/drain_decide.go new file mode 100644 index 0000000000..ed0d33b1b1 --- /dev/null +++ b/internal/core/implement/loop/drain_decide.go @@ -0,0 +1,166 @@ +package loop + +// drain_decide.go is the decision adapter at the drain's judgement +// (itd-2609221009495079, spc-2609221011151661 scope 2). The judgement a drain +// asks before it opens an issue's lane is a closed-option question over one +// record — no, or yes and which kind of hand-back — so it is the judgement +// type eligibility-residual: the residual of the drain's eligibility once the +// field rule has passed, and it is routed through core/decide/judge, as every +// typed judgement is. +// +// - Configured for shadow, the host (or the routed runner) decides exactly +// as it does today; once its answer is taken, the adapter is asked the +// same question over the same record and the pair is recorded in the +// newest lab home (shadowDrainJudgement). Shadow never changes, delays or +// stops the drain. +// - Turned on, the adapter is asked once the host's request is written +// (decideDrainJudgement): its answer is taken as a routed judge's is, +// under the same disposition rules (judgeAnswer), and the judgement's +// Route names the adapter and the model that answered. An adapter that +// cannot answer leaves the host's request standing, with a fallback +// receipt kept beside the routed judge's. +// +// Both functions read the configuration themselves (loadDrainRouter), so the +// drain's call sites pass only what they already hold. + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/intentdriven/abcd/internal/core/capture" + decision "github.com/intentdriven/abcd/internal/core/decide/judge" + "github.com/intentdriven/abcd/internal/core/oracle" + "github.com/intentdriven/abcd/internal/core/runner" + "github.com/intentdriven/abcd/internal/termsafe" +) + +// drainJudgementType is the drain's judgement in the decision adapter's +// registry. +const drainJudgementType = decision.EligibilityResidual + +// drainAdapterRoute names the decision adapter in a judgement's Route and a +// fallback receipt's Asked, beside its /. +const drainAdapterRoute = "decision-adapter" + +// drainAdapterTimeout bounds one adapter call: a typed answer over one record +// takes about a second, so this is for a provider that hangs. +const drainAdapterTimeout = 2 * time.Minute + +// loadDrainRouter reads the decision adapter's configuration for the checkout +// at repoRoot. Tests replace it. +var loadDrainRouter = func(repoRoot string, o Options) (*decision.Router, error) { + return decision.Load(o.roots(repoRoot), repoRoot) +} + +// drainQuestion is the drain's one question as a typed judgement: no, or the +// kind of hand-back a yes names. +func drainQuestion() decision.Question { + return decision.Question{Type: drainJudgementType, Kind: decision.KindChoose, + Ask: "Does this remedy, carried out as written, change what a user sees, or a trust boundary? " + + "What a user sees is anything a person reads or relies on: a command's output or exit code, a flag, " + + "a message, a page, a prompt, or what a command does when a person runs it. A trust boundary is what is " + + "allowed or refused, read, written, run or sent, whose input is trusted, or a rule about safety, secrets " + + "or permissions. Answer " + HandBackUserVisible + " when it changes what a user sees, " + HandBackTrustRule + + " when it changes a trust boundary (or both, or you cannot tell), and no only when it changes neither.", + Options: []string{JudgementNo, HandBackUserVisible, HandBackTrustRule}} +} + +// drainState is the state the judgement is made over: the issue as the +// host's request shows it, without the request's instructions about where to +// write the answer. +func drainState(iss capture.Issue) decision.State { + var b strings.Builder + fmt.Fprintf(&b, "Issue %s\nseverity: %s\ncategory: %s\n\nRemedy:\n%s\n\nRecord body:\n%s\n", + iss.ID, termsafe.Sanitize(string(iss.Severity)), termsafe.Sanitize(string(iss.Category)), + termsafe.Sanitize(iss.Remedy), termsafe.SanitizeBlock(strings.TrimSpace(iss.Body))) + return decision.State{Text: b.String()} +} + +// drainValue is a taken judgement as the question's choice: no, or the kind. +func drainValue(j DrainJudgement) string { + if j.Answer == JudgementNo { + return JudgementNo + } + return j.Kind +} + +// shadowDrainJudgement asks the adapter beside a judgement the drain has +// taken, j, when eligibility-residual is configured for shadow, and records +// the pair. It returns the receipt, nil when the type is not in shadow, the +// issue has left the open ledger, or the configuration cannot be read (the +// drain has its answer; shadow never stops it). The drain's take point calls +// it before applying j. +func shadowDrainJudgement(ctx context.Context, repoRoot string, o Options, j DrainJudgement, plan capture.DrainPlan) *decision.Receipt { + iss, ok := plan.OpenIssue(j.Issue) + if !ok || drainValue(j) == "" { + return nil + } + r, err := loadDrainRouter(repoRoot, o) + if err != nil || r.Mode(drainJudgementType) != oracle.JudgementShadow { + return nil + } + ctx, cancel := context.WithTimeout(ctxOr(ctx), drainAdapterTimeout) + defer cancel() + _, rc, err := r.Decide(ctx, drainState(iss), drainQuestion(), decision.Answered(drainValue(j))) + if err != nil { + return nil + } + return &rc +} + +// decideDrainJudgement asks the adapter for the judgement the drain awaits, +// judging, when eligibility-residual is turned on. It returns the judgement +// the adapter decided, taken as a routed judge's answer is (judgeAnswer), with +// its Route naming the adapter; or nil and the fallback receipt when the +// adapter could not answer, the host's request then standing; or nil, nil when +// the type is not turned on. A configuration the oracle reader refuses (a +// model off its provider's list) is an error, since a route the person set is +// never dropped silently; nothing was called. +func decideDrainJudgement(ctx context.Context, repoRoot string, o Options, judging DrainJudging, plan capture.DrainPlan, + now time.Time) (*DrainJudgement, *runner.FallbackReceipt, error) { + iss, ok := plan.OpenIssue(judging.Issue) + if !ok { + return nil, nil, nil + } + r, err := loadDrainRouter(repoRoot, o) + if err != nil { + return nil, nil, fmt.Errorf("the decision adapter's configuration: %w", err) + } + if r.Mode(drainJudgementType) != oracle.JudgementOn { + return nil, nil, nil + } + ctx, cancel := context.WithTimeout(ctxOr(ctx), drainAdapterTimeout) + defer cancel() + a, rc, err := r.Decide(ctx, drainState(iss), drainQuestion(), decision.Pending()) + asked := drainAdapterRoute + " " + rc.Target + if errors.Is(err, decision.ErrHostPending) { + return nil, &runner.FallbackReceipt{At: now, Role: RoleDrainJudge, Asked: asked, Reason: runner.ReasonFailed, + Detail: rc.Fallback + "; the host's request stands", Ran: runner.Host}, nil + } + if err != nil { + return nil, nil, err + } + ans := JudgementAnswer{SchemaVersion: DrainJudgementSchemaVersion, Issue: judging.Issue, RemedySHA256: judging.RemedySHA256, + Answer: JudgementYes, Kind: a.Value, + Reason: fmt.Sprintf("the decision adapter %s answered %s at probability %.2f", rc.Target, a.Value, a.Probability)} + if a.Value == JudgementNo { + ans.Answer, ans.Kind = JudgementNo, "" + } + taken, err := judgeAnswer(ans, judging, plan, now) + if err != nil { + return nil, nil, err + } + taken.Route = &runner.RouteRecord{Asked: asked, Ran: asked, Model: termsafe.Sanitize(rc.ModelReported)} + return &taken, nil, nil +} + +// ctxOr is ctx, or the background context when it is nil. +func ctxOr(ctx context.Context) context.Context { + if ctx == nil { + return context.Background() + } + return ctx +} diff --git a/internal/core/implement/loop/drain_decide_test.go b/internal/core/implement/loop/drain_decide_test.go new file mode 100644 index 0000000000..2a939d9c73 --- /dev/null +++ b/internal/core/implement/loop/drain_decide_test.go @@ -0,0 +1,174 @@ +package loop + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/intentdriven/abcd/internal/abcdhome" + "github.com/intentdriven/abcd/internal/adapter/openaiapi" + "github.com/intentdriven/abcd/internal/core/capture" + decision "github.com/intentdriven/abcd/internal/core/decide/judge" + "github.com/intentdriven/abcd/internal/core/lab" + "github.com/intentdriven/abcd/internal/core/layered" + "github.com/intentdriven/abcd/internal/core/oracle" + "github.com/intentdriven/abcd/internal/core/runner" +) + +// The drain's judgement through the decision adapter (itd-2609221009495079): +// eligibility-residual in shadow records a pair beside the host's answer and +// changes nothing; turned on, the adapter's answer is taken as a routed +// judge's is, and one it cannot give leaves the host's request standing with +// a fallback receipt. + +type drainFakeJudge struct { + value string + p float64 + err error + calls int +} + +func (f *drainFakeJudge) Judge(context.Context, decision.State, decision.Question) (decision.Answer, error) { + f.calls++ + if f.err != nil { + return decision.Answer{}, f.err + } + return decision.Answer{Value: f.value, Probability: f.p, Model: "typesafe/jev-1.13-0901"}, nil +} + +// withDrainRouter makes the drain judge with eligibility-residual in mode, +// adapter answering, and pairs kept in got. +func withDrainRouter(t *testing.T, mode string, adapter *drainFakeJudge, got *[]lab.Pair) { + t.Helper() + routes := map[decision.Type]oracle.Target{} + if mode != oracle.JudgementHost { + routes[decision.EligibilityResidual] = oracle.Target{Provider: "openrouter", Model: "typesafe/jev-1.13", Mode: mode} + } + prev := loadDrainRouter + loadDrainRouter = func(string, Options) (*decision.Router, error) { + return decision.New(routes, func(oracle.Target) decision.Judge { return adapter }, func(p lab.Pair) (string, string, error) { + *got = append(*got, p) + return "lab-261029000000-abcdef0", "~/.abcd.noindex/lab/x/lab-261029000000-abcdef0/pairs.jsonl", nil + }), nil + } + t.Cleanup(func() { loadDrainRouter = prev }) +} + +func drainDecidePlan(t *testing.T) (*capture.DrainPlan, string, DrainJudging) { + t.Helper() + repo := drainRepo(t) + plan, err := capture.PlanDrain(capture.DrainPlanRequest{RepoRoot: repo.Root()}) + if err != nil { + t.Fatal(err) + } + iss, ok := plan.OpenIssue(eligibleIssue) + if !ok { + t.Fatalf("%s is not open in the plan", eligibleIssue) + } + return &plan, repo.Root(), DrainJudging{Issue: eligibleIssue, RemedySHA256: remedyDigest(iss.Remedy)} +} + +func TestShadowDrainJudgementRecordsThePairBesideTheHostsAnswer(t *testing.T) { + plan, root, _ := drainDecidePlan(t) + adapter, got := &drainFakeJudge{value: HandBackTrustRule, p: 0.8}, []lab.Pair{} + withDrainRouter(t, oracle.JudgementShadow, adapter, &got) + j := DrainJudgement{Issue: eligibleIssue, Answer: JudgementNo, Reason: "a renderer guard", Applied: true} + rc := shadowDrainJudgement(context.Background(), root, Options{}, j, *plan) + if rc == nil || rc.Route != decision.RouteHost || rc.Type != decision.EligibilityResidual || !strings.HasSuffix(rc.Pair, "pairs.jsonl") { + t.Fatalf("receipt = %+v", rc) + } + if len(got) != 1 || got[0].Host != JudgementNo || got[0].Adapter != HandBackTrustRule || *got[0].Probability != 0.8 || + got[0].Type != string(decision.EligibilityResidual) { + t.Fatalf("pairs = %+v", got) + } + // A yes is paired as its kind. + j = DrainJudgement{Issue: eligibleIssue, Answer: JudgementYes, Kind: HandBackUserVisible, Reason: "a flag"} + shadowDrainJudgement(context.Background(), root, Options{}, j, *plan) + if len(got) != 2 || got[1].Host != HandBackUserVisible { + t.Fatalf("pairs = %+v", got) + } +} + +func TestShadowDrainJudgementDoesNothingOffShadow(t *testing.T) { + plan, root, _ := drainDecidePlan(t) + for _, mode := range []string{oracle.JudgementHost, oracle.JudgementOn} { + adapter, got := &drainFakeJudge{value: JudgementNo, p: 0.9}, []lab.Pair{} + withDrainRouter(t, mode, adapter, &got) + if rc := shadowDrainJudgement(context.Background(), root, Options{}, DrainJudgement{Issue: eligibleIssue, Answer: JudgementNo}, *plan); rc != nil || adapter.calls != 0 || len(got) != 0 { + t.Fatalf("%s: receipt %+v, adapter calls %d, pairs %d", mode, rc, adapter.calls, len(got)) + } + } +} + +func TestDecideDrainJudgementTakesTheAdaptersAnswerWhenTurnedOn(t *testing.T) { + plan, root, judging := drainDecidePlan(t) + adapter, got := &drainFakeJudge{value: HandBackUserVisible, p: 0.91}, []lab.Pair{} + withDrainRouter(t, oracle.JudgementOn, adapter, &got) + now := time.Date(2026, 10, 29, 6, 0, 0, 0, time.UTC) + taken, fb, err := decideDrainJudgement(context.Background(), root, Options{}, judging, *plan, now) + if err != nil || fb != nil || taken == nil { + t.Fatalf("taken %+v, fallback %+v, err %v", taken, fb, err) + } + if taken.Answer != JudgementYes || taken.Kind != HandBackUserVisible || !taken.Applied || !taken.At.Equal(now) || + !strings.Contains(taken.Reason, "probability 0.91") { + t.Fatalf("taken = %+v", taken) + } + if r := taken.Route; r == nil || r.Asked != "decision-adapter openrouter/typesafe/jev-1.13" || r.Ran != r.Asked || r.Model != "typesafe/jev-1.13-0901" { + t.Fatalf("route = %+v: the run record names the route", taken.Route) + } + // A no is taken as a no, and opens the lane. + adapter.value = JudgementNo + taken, _, _ = decideDrainJudgement(context.Background(), root, Options{}, judging, *plan, now) + if taken == nil || taken.Answer != JudgementNo || taken.Kind != "" || !taken.Applied { + t.Fatalf("no = %+v", taken) + } +} + +func TestDecideDrainJudgementFallsBackToTheHostWithAReceipt(t *testing.T) { + plan, root, judging := drainDecidePlan(t) + adapter, got := &drainFakeJudge{err: openaiapi.ErrUnreachable}, []lab.Pair{} + withDrainRouter(t, oracle.JudgementOn, adapter, &got) + taken, fb, err := decideDrainJudgement(context.Background(), root, Options{}, judging, *plan, time.Now()) + if err != nil || taken != nil || fb == nil { + t.Fatalf("taken %+v, fallback %+v, err %v", taken, fb, err) + } + if fb.Role != RoleDrainJudge || fb.Ran != runner.Host || !strings.Contains(fb.Asked, "openrouter/typesafe/jev-1.13") || + !strings.Contains(fb.Detail, "the host's request stands") { + t.Fatalf("fallback = %+v", fb) + } +} + +func TestDecideDrainJudgementAsksNothingUnlessTurnedOn(t *testing.T) { + plan, root, judging := drainDecidePlan(t) + for _, mode := range []string{oracle.JudgementHost, oracle.JudgementShadow} { + adapter, got := &drainFakeJudge{value: JudgementNo, p: 0.9}, []lab.Pair{} + withDrainRouter(t, mode, adapter, &got) + taken, fb, err := decideDrainJudgement(context.Background(), root, Options{}, judging, *plan, time.Now()) + if taken != nil || fb != nil || err != nil || adapter.calls != 0 { + t.Fatalf("%s: taken %+v fallback %+v err %v calls %d", mode, taken, fb, err, adapter.calls) + } + } +} + +// The real configuration: a model off its provider's list refuses the +// judgement before any call, naming the list. +func TestDecideDrainJudgementRefusesAnUnlistedModelBeforeAnyCall(t *testing.T) { + plan, root, judging := drainDecidePlan(t) + home := t.TempDir() + cfg := abcdhome.Path(home, "config.json") + if err := os.MkdirAll(filepath.Dir(cfg), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(cfg, []byte(`{"oracle":{"api":{"openrouter":{"base_url":"https://openrouter.ai/api/v1","key":"openrouter", + "models":["typesafe/jev-1.13"]}},"judgements":{"eligibility-residual":"openrouter/mistral/small"}}}`), 0o600); err != nil { + t.Fatal(err) + } + o := Options{Roots: &layered.Roots{Repo: root, Home: home}} + _, _, err := decideDrainJudgement(context.Background(), root, o, judging, *plan, time.Now()) + if err == nil || !strings.Contains(err.Error(), "not on openrouter's list") { + t.Fatalf("err = %v, want the allowlist refusal", err) + } +} From 3909542215b01381a8621069e3b6a50c8f983175 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:03:07 +0000 Subject: [PATCH 09/20] fix: keep the judge package's surface to what production code reaches The reach audit refuses an exported core function no production code outside its package calls: render, known and types are unexported, the router's constructor is newRouter with NewForTest as the loop tests' seam, and the unused Routes method is gone. The tests build the abcd home's paths through abcdhome.Display instead of spelling the folder. Assisted-by: Claude:claude-opus-5-5 --- internal/core/decide/judge/adapter.go | 2 +- internal/core/decide/judge/host.go | 2 +- internal/core/decide/judge/judge.go | 14 ++++---- internal/core/decide/judge/judge_test.go | 18 +++++----- internal/core/decide/judge/router.go | 33 ++++++++----------- .../core/implement/loop/drain_decide_test.go | 4 +-- 6 files changed, 33 insertions(+), 40 deletions(-) diff --git a/internal/core/decide/judge/adapter.go b/internal/core/decide/judge/adapter.go index b59ce73b91..54dff2d29e 100644 --- a/internal/core/decide/judge/adapter.go +++ b/internal/core/decide/judge/adapter.go @@ -62,7 +62,7 @@ func (a Adapter) Judge(ctx context.Context, s State, q Question) (Answer, error) got = v return err } - brief := openaiapi.Brief{Instructions: adapterInstructions, Input: Render(s, q)} + brief := openaiapi.Brief{Instructions: adapterInstructions, Input: render(s, q)} _, rec, err := a.Calls.Call(ctx, a.Creds, oracle.CallRequest{Target: a.Target, Brief: brief, Contract: contract}, a.Options...) if err != nil { return Answer{}, err diff --git a/internal/core/decide/judge/host.go b/internal/core/decide/judge/host.go index a48719ae4a..eb0436a7fb 100644 --- a/internal/core/decide/judge/host.go +++ b/internal/core/decide/judge/host.go @@ -40,7 +40,7 @@ func (h Host) Judge(ctx context.Context, s State, q Question) (Answer, error) { if h.Ask == nil { return Answer{}, ErrHostPending } - text, err := h.Ask(ctx, Render(s, q)) + text, err := h.Ask(ctx, render(s, q)) if err != nil { return Answer{}, err } diff --git a/internal/core/decide/judge/judge.go b/internal/core/decide/judge/judge.go index 9184823894..f89d9554de 100644 --- a/internal/core/decide/judge/judge.go +++ b/internal/core/decide/judge/judge.go @@ -62,11 +62,11 @@ const ( var registry = []Type{RemedyRead, DuplicateMatch, UnmeetableCriteria, EligibilityResidual, SourceClassification, SourceSelection, MemoryCheck, LedgerCategory, LedgerSeverity, PrepassOverlap, PresentTense} -// Types returns the registry, in its order. -func Types() []Type { return slices.Clone(registry) } +// types returns the registry, in its order. +func types() []Type { return slices.Clone(registry) } -// Known reports whether name is a type in the registry. -func Known(name string) bool { return slices.Contains(registry, Type(name)) } +// known reports whether name is a type in the registry. +func known(name string) bool { return slices.Contains(registry, Type(name)) } // Bounds on a question. const ( @@ -107,7 +107,7 @@ func (q Question) Choices() []string { // Check refuses a question that does not hold together: a type outside the // registry, an unknown kind, no words, or options or a scale out of shape. func (q Question) Check() error { - if !Known(string(q.Type)) { + if !known(string(q.Type)) { return fmt.Errorf("judge: %q is not a judgement type (the registry is %s)", clip(string(q.Type)), typeNames()) } if strings.TrimSpace(q.Ask) == "" || len(q.Ask) > maxAsk { @@ -175,9 +175,9 @@ type Judge interface { Judge(ctx context.Context, s State, q Question) (Answer, error) } -// Render is the prompt a judgement is asked as, the same for the host and the +// render is the prompt a judgement is asked as, the same for the host and the // adapter. -func Render(s State, q Question) string { +func render(s State, q Question) string { var b strings.Builder fmt.Fprintf(&b, "Judgement: %s (%s)\n\nQuestion: %s\n\n", q.Type, q.Kind, strings.TrimSpace(q.Ask)) switch q.Kind { diff --git a/internal/core/decide/judge/judge_test.go b/internal/core/decide/judge/judge_test.go index 26cbcd9f24..f4c85c9bcc 100644 --- a/internal/core/decide/judge/judge_test.go +++ b/internal/core/decide/judge/judge_test.go @@ -24,7 +24,7 @@ import ( // same interface; TestTheAPIAdapterParsesTheTypedAnswer drives the real // Adapter over a fake oracle caller. -var target = oracle.Target{Provider: "openrouter", Model: "typesafe/jev-1.13", Origin: "~/.abcd.noindex/config.json"} +var target = oracle.Target{Provider: "openrouter", Model: "typesafe/jev-1.13", Origin: abcdhome.Display("config.json")} func dupQuestion() Question { return Question{Type: DuplicateMatch, Kind: KindChoose, Ask: "Which record does this capture duplicate?", @@ -42,7 +42,7 @@ type fake struct { func (f *fake) Judge(_ context.Context, s State, q Question) (Answer, error) { f.calls++ - f.seen = append(f.seen, Render(s, q)) + f.seen = append(f.seen, render(s, q)) if f.err != nil { return Answer{}, f.err } @@ -60,7 +60,7 @@ func (p *pairs) record(pr lab.Pair) (string, string, error) { return "", "", p.err } p.got = append(p.got, pr) - return "lab-261029000000-abcdef0", "~/.abcd.noindex/lab/x/lab-261029000000-abcdef0/pairs.jsonl", nil + return "lab-261029000000-abcdef0", abcdhome.Display("lab/x/lab-261029000000-abcdef0/pairs.jsonl"), nil } func router(mode string, adapter *fake, rec *pairs) *Router { @@ -70,7 +70,7 @@ func router(mode string, adapter *fake, rec *pairs) *Router { t.Mode = mode routes[DuplicateMatch] = t } - return New(routes, func(oracle.Target) Judge { return adapter }, rec.record) + return newRouter(routes, func(oracle.Target) Judge { return adapter }, rec.record) } // Criterion 1: shadow — the host decides, the adapter answers beside it, and @@ -117,7 +117,7 @@ func TestShadowWritesThePairUnderTheLabHomeAndNothingInTheRepository(t *testing. before := r.Git("status", "--porcelain", "--ignored") t2 := target t2.Mode = oracle.JudgementShadow - rt := New(map[Type]oracle.Target{DuplicateMatch: t2}, func(oracle.Target) Judge { return &fake{value: "none", p: 0.7} }, + rt := newRouter(map[Type]oracle.Target{DuplicateMatch: t2}, func(oracle.Target) Judge { return &fake{value: "none", p: 0.7} }, func(p lab.Pair) (string, string, error) { return lab.RecordPair(r.Root(), p) }) if _, _, err := rt.Decide(context.Background(), State{Text: "s"}, dupQuestion(), Answered("none")); err != nil { t.Fatal(err) @@ -271,12 +271,12 @@ func TestTheRegistryIsClosedAndAQuestionIsChecked(t *testing.T) { if host.calls != 0 { t.Fatal("the host was asked a refused question") } - for _, ty := range Types() { - if !Known(string(ty)) { + for _, ty := range types() { + if !known(string(ty)) { t.Errorf("%s is listed but not known", ty) } } - if Known("embark-lesson-ranking") { + if known("embark-lesson-ranking") { t.Error("embark's lesson ranking is a term-overlap computation, not a judgement (iss-2610262033055327)") } } @@ -321,7 +321,7 @@ func TestTheHostParsesItsAnswerAtProbabilityOne(t *testing.T) { if _, err := h.Judge(context.Background(), State{Text: "the state"}, dupQuestion()); err != nil { t.Fatal(err) } - if asked != Render(State{Text: "the state"}, dupQuestion()) || !strings.Contains(asked, "the state") || !strings.Contains(asked, "iss-2") { + if asked != render(State{Text: "the state"}, dupQuestion()) || !strings.Contains(asked, "the state") || !strings.Contains(asked, "iss-2") { t.Fatalf("host prompt = %q", asked) } } diff --git a/internal/core/decide/judge/router.go b/internal/core/decide/judge/router.go index da2b0669c2..95d14f1905 100644 --- a/internal/core/decide/judge/router.go +++ b/internal/core/decide/judge/router.go @@ -23,7 +23,6 @@ import ( "errors" "fmt" "math" - "sort" "strings" "github.com/intentdriven/abcd/internal/core/credential" @@ -83,9 +82,10 @@ type Router struct { Diagnostics []string } -// New is a Router over routes (a type absent from it stays on the host), the -// adapter each target is judged by, and where shadow pairs are recorded. -func New(routes map[Type]oracle.Target, adapter func(oracle.Target) Judge, record Recorder) *Router { +// newRouter is a Router over routes (a type absent from it stays on the +// host), the adapter each target is judged by, and where shadow pairs are +// recorded. +func newRouter(routes map[Type]oracle.Target, adapter func(oracle.Target) Judge, record Recorder) *Router { r := &Router{routes: map[Type]oracle.Target{}, adapter: adapter, record: record} for t, tgt := range routes { r.routes[t] = tgt @@ -93,6 +93,12 @@ func New(routes map[Type]oracle.Target, adapter func(oracle.Target) Judge, recor return r } +// NewForTest is newRouter, the seam a caller's tests route a type with +// through a fake adapter and recorder. +func NewForTest(routes map[Type]oracle.Target, adapter func(oracle.Target) Judge, record Recorder) *Router { + return newRouter(routes, adapter, record) +} + // Load reads oracle.judgements. through the oracle reader, which refuses // a model not on its provider's allowlist (or one oracle.denylist matches) // before any call, and returns the Router the repository at repoRoot judges @@ -111,7 +117,7 @@ func Load(roots layered.Roots, repoRoot string) (*Router, error) { if pr.Kind != "judgement" { continue } - if !Known(pr.Name) { + if !known(pr.Name) { diags = append(diags, fmt.Sprintf("decision adapter: oracle.judgements.%s in %s is not a judgement type (%s); it is skipped", termsafe.Sanitize(clip(pr.Name)), termsafe.Sanitize(pr.Target.Origin), typeNames())) continue @@ -119,7 +125,7 @@ func Load(roots layered.Roots, repoRoot string) (*Router, error) { routes[Type(pr.Name)] = pr.Target } creds := credential.Store(roots.Home) - r := New(routes, func(t oracle.Target) Judge { return Adapter{Calls: cfg, Creds: creds, Target: t} }, + r := newRouter(routes, func(t oracle.Target) Judge { return Adapter{Calls: cfg, Creds: creds, Target: t} }, func(p lab.Pair) (string, string, error) { return lab.RecordPair(repoRoot, p) }) r.Diagnostics = diags return r, nil @@ -150,7 +156,7 @@ func (r *Router) Decide(ctx context.Context, s State, q Question, host Judge) (A return Answer{}, Receipt{}, err } mode := r.Mode(q.Type) - size := tokens(Render(s, q)) + size := tokens(render(s, q)) rc := Receipt{Type: q.Type, Mode: mode, Route: RouteHost, StateSHA256: s.SHA256(), StateTokens: size} tgt := r.routes[q.Type] if mode != oracle.JudgementHost { @@ -240,19 +246,6 @@ func (r *Router) ask(ctx context.Context, tgt oracle.Target, s State, q Question return a, nil } -// Routes are the types routed off the host, sorted, with their routes as the -// configuration spells them, for a board. -func (r *Router) Routes() []string { - var out []string - for t, tgt := range r.routes { - if m := r.Mode(t); m != oracle.JudgementHost { - out = append(out, string(t)+" = "+tgt.Route()) - } - } - sort.Strings(out) - return out -} - // detail is an error's account for a pair or a receipt: terminal-safe, one // line, bounded. func detail(err error) string { diff --git a/internal/core/implement/loop/drain_decide_test.go b/internal/core/implement/loop/drain_decide_test.go index 2a939d9c73..19140b23a3 100644 --- a/internal/core/implement/loop/drain_decide_test.go +++ b/internal/core/implement/loop/drain_decide_test.go @@ -49,9 +49,9 @@ func withDrainRouter(t *testing.T, mode string, adapter *drainFakeJudge, got *[] } prev := loadDrainRouter loadDrainRouter = func(string, Options) (*decision.Router, error) { - return decision.New(routes, func(oracle.Target) decision.Judge { return adapter }, func(p lab.Pair) (string, string, error) { + return decision.NewForTest(routes, func(oracle.Target) decision.Judge { return adapter }, func(p lab.Pair) (string, string, error) { *got = append(*got, p) - return "lab-261029000000-abcdef0", "~/.abcd.noindex/lab/x/lab-261029000000-abcdef0/pairs.jsonl", nil + return "lab-261029000000-abcdef0", abcdhome.Display("lab/x/lab-261029000000-abcdef0/pairs.jsonl"), nil }), nil } t.Cleanup(func() { loadDrainRouter = prev }) From f29b0aeb0a903daa23f72074357113f3c42dddd8 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:09:02 +0000 Subject: [PATCH 10/20] Revert "docs: drop embark's lesson ranking from the decision adapter's judgement types" This reverts commit 966013a1198df9743fa14b16462924af331ddf0b. The product thinker has ruled that embark's lesson ranking stays in the decision adapter's scope. It becomes a judgement routed through the adapter's one interface, replacing today's term-overlap ordering, rather than a computation left outside the adapter. The facilitator-on-delegation decision recorded in the reverted commit, which dropped the ranking as a factual correction, is withdrawn. The intent's two lists name the ranking again, and iss-2610262033055327 returns to open/. It now stands for the work of turning the ranking into that judgement. Refs: iss-2610262033055327 Assisted-by: Claude:claude-opus-5-5 --- ...09221009495079-abcd-s-typed-judgements-can-run-through.md | 4 ++-- ...ss-2610262033055327-the-decision-adapter-intent-treats.md | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) rename .abcd/work/issues/{resolved => open}/iss-2610262033055327-the-decision-adapter-intent-treats.md (86%) diff --git a/.abcd/development/intents/planned/itd-2609221009495079-abcd-s-typed-judgements-can-run-through.md b/.abcd/development/intents/planned/itd-2609221009495079-abcd-s-typed-judgements-can-run-through.md index d18990bea1..d92a6a439b 100644 --- a/.abcd/development/intents/planned/itd-2609221009495079-abcd-s-typed-judgements-can-run-through.md +++ b/.abcd/development/intents/planned/itd-2609221009495079-abcd-s-typed-judgements-can-run-through.md @@ -25,7 +25,7 @@ related_adrs: [adr-2609221009491186] ## Why This Matters -The research pass of 2026-09-22 found Jev (TypeSafe AI) to be a typed-decision model, not a text model and not a router: choose one of N, score against a rubric, yes or no, each with a calibrated probability, in under a second at $0.042 per million tokens, 32K context, and unable to abstain. Independent evidence is one forty-case test; the vendor's numbers are vendor-graded. abcd's judgement-shaped steps are many: the remedy read, duplicate matching, unmeetable criteria, the eligibility residual, source classification and selection in the library, memory checks, ledger category and severity, the pre-pass's overlap, the docs lint's present-tense advisory. Ruled: an adapter behind one interface, the host as default, shadow mode as an abcd lab, a research note as the evidence a ruling turns a judgement type on with. +The research pass of 2026-09-22 found Jev (TypeSafe AI) to be a typed-decision model, not a text model and not a router: choose one of N, score against a rubric, yes or no, each with a calibrated probability, in under a second at $0.042 per million tokens, 32K context, and unable to abstain. Independent evidence is one forty-case test; the vendor's numbers are vendor-graded. abcd's judgement-shaped steps are many: the remedy read, duplicate matching, unmeetable criteria, the eligibility residual, source classification and selection in the library, memory checks, ledger category and severity, the pre-pass's overlap, the docs lint's present-tense advisory, embark's lesson ranking. Ruled: an adapter behind one interface, the host as default, shadow mode as an abcd lab, a research note as the evidence a ruling turns a judgement type on with. ## Mechanism @@ -40,7 +40,7 @@ We expect calibrated typed answers to agree with the host on closed-option judge ## What's In Scope - **One interface**: `(state, typed question) → (value, probability)` with three question kinds (choose, score, yes/no); the host's own judgement implements it too and stays the default. -- **The judgement types** it may be configured for: the remedy read, duplicate matching, unmeetable criteria, the eligibility residual, source classification and selection over a shortlist, the memory checks, ledger category and severity, the pre-pass's overlap, the docs lint's present-tense advisory. Embark's lesson ranking is not one: the embark page ranks lessons by term overlap against the new voyage's brief, a computation with no host answer to pair (iss-2610262033055327). +- **The judgement types** it may be configured for: the remedy read, duplicate matching, unmeetable criteria, the eligibility residual, source classification and selection over a shortlist, the memory checks, ledger category and severity, the pre-pass's overlap, the docs lint's present-tense advisory, embark's lesson ranking. - **Shadow mode as a lab**: `abcd lab mint` opens a lab home; the adapter runs beside the host on the configured types, decides nothing, and each pair (host's answer, adapter's answer, probability, the state's hash) is recorded under the lab home in the machine-scoped store, never the repository. - **The evidence**: `abcd lab harvest` produces agreement rates per type with counts and probability bands; the research note filed from it (dated, under the record's research notes) is the evidence a ruling turns a type on with. - **Turning on**: per judgement type in configuration; unreachable falls back to the host with a receipt; the run record names which route judged. diff --git a/.abcd/work/issues/resolved/iss-2610262033055327-the-decision-adapter-intent-treats.md b/.abcd/work/issues/open/iss-2610262033055327-the-decision-adapter-intent-treats.md similarity index 86% rename from .abcd/work/issues/resolved/iss-2610262033055327-the-decision-adapter-intent-treats.md rename to .abcd/work/issues/open/iss-2610262033055327-the-decision-adapter-intent-treats.md index 6e76d9b82b..06a7808c38 100644 --- a/.abcd/work/issues/resolved/iss-2610262033055327-the-decision-adapter-intent-treats.md +++ b/.abcd/work/issues/open/iss-2610262033055327-the-decision-adapter-intent-treats.md @@ -12,11 +12,6 @@ found_at: ".abcd/development/intents/planned/itd-2609221009495079-abcd-s-typed-j remedy: "In the decision-adapter intent drop embark's lesson ranking from the judgement-type lists (press-release rationale and scope bullet), or add a host judgement for it first: the shipped ranking is a deterministic term-overlap computation with no host answer to pair." related_intents: [itd-2609221009495079] finding_ref: cfk-0fa68fc1ec4b7f88 -resolution: "embark's lesson ranking is dropped from the decision adapter intent's judgement types and is not in the judge package's registry: the embark page ranks lessons by term overlap, a computation with no host answer to pair" -impact: internal -resolved_by: - intent: "itd-2609221009495079" - spec: "spc-2609221011151661" --- The decision-adapter intent treats embark's lesson ranking as a judgement the host answers by default; the embark page ranks lessons by term overlap, with no judgement in that step. From 8f1333050473f4fcfc1fcbfb5dbcba955d779402 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 29 Oct 2026 09:27:43 +0000 Subject: [PATCH 11/20] feat: register embark's lesson ranking as a decision adapter judgement type Under the product thinker's ruling, embark's lesson ranking is a judgement routed through the adapter in place of its term-overlap ordering, so embark-lesson-ranking joins the closed registry and the configuration chapter's list. The embark verb does not call the router yet; until it does, the ranking stays as it is. Refs: iss-2610262033055327 Assisted-by: Claude:claude-opus-5-5 --- .../brief/05-internals/03-configuration.md | 3 ++- internal/core/decide/judge/judge.go | 10 ++++++---- internal/core/decide/judge/judge_test.go | 12 +++++++----- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.abcd/development/brief/05-internals/03-configuration.md b/.abcd/development/brief/05-internals/03-configuration.md index 1363bca41c..735c702941 100644 --- a/.abcd/development/brief/05-internals/03-configuration.md +++ b/.abcd/development/brief/05-internals/03-configuration.md @@ -359,7 +359,8 @@ rather than skipped: The types are a closed registry: `remedy-read`, `duplicate-match`, `unmeetable-criteria`, `eligibility-residual`, `source-classification`, `source-selection`, `memory-check`, `ledger-category`, `ledger-severity`, - `prepass-overlap`, `present-tense-advisory`. A route naming anything else is + `prepass-overlap`, `present-tense-advisory`, `embark-lesson-ranking`. A + route naming anything else is a diagnostic and is skipped. The drain's judgement before a lane opens is meant to run as `eligibility-residual`, a choice of `no`, `user-visible` or `trust-rule`. Its routing is built (`loop/drain_decide.go`). In shadow, the diff --git a/internal/core/decide/judge/judge.go b/internal/core/decide/judge/judge.go index f89d9554de..50df3e16fa 100644 --- a/internal/core/decide/judge/judge.go +++ b/internal/core/decide/judge/judge.go @@ -42,9 +42,10 @@ const ( // the adapter. The registry is closed: a type outside it is refused. type Type string -// The judgement types the intent's scope lists. Embark's lesson ranking is not -// one: the embark page ranks lessons by term overlap, a computation with no -// host answer to pair (iss-2610262033055327). +// The judgement types the intent's scope lists. Embark's lesson ranking is +// one: it is to run through the adapter in place of the term-overlap ordering +// the embark verb uses today (iss-2610262033055327), and like every other +// type it is decided by the host until its step calls the router. const ( RemedyRead Type = "remedy-read" DuplicateMatch Type = "duplicate-match" @@ -57,10 +58,11 @@ const ( LedgerSeverity Type = "ledger-severity" PrepassOverlap Type = "prepass-overlap" PresentTense Type = "present-tense-advisory" + EmbarkLessonRanking Type = "embark-lesson-ranking" ) var registry = []Type{RemedyRead, DuplicateMatch, UnmeetableCriteria, EligibilityResidual, SourceClassification, - SourceSelection, MemoryCheck, LedgerCategory, LedgerSeverity, PrepassOverlap, PresentTense} + SourceSelection, MemoryCheck, LedgerCategory, LedgerSeverity, PrepassOverlap, PresentTense, EmbarkLessonRanking} // types returns the registry, in its order. func types() []Type { return slices.Clone(registry) } diff --git a/internal/core/decide/judge/judge_test.go b/internal/core/decide/judge/judge_test.go index f4c85c9bcc..d2fd7a3478 100644 --- a/internal/core/decide/judge/judge_test.go +++ b/internal/core/decide/judge/judge_test.go @@ -262,7 +262,7 @@ func TestOneInterfaceSwitchesATypeWithoutTheCallerChanging(t *testing.T) { func TestTheRegistryIsClosedAndAQuestionIsChecked(t *testing.T) { rt := router(oracle.JudgementOn, &fake{value: "x"}, &pairs{}) host := &fake{value: "none"} - if _, _, err := rt.Decide(context.Background(), State{Text: "s"}, Question{Type: "lesson-ranking", Kind: KindChoose, Ask: "?", Options: []string{"a"}}, host); err == nil { + if _, _, err := rt.Decide(context.Background(), State{Text: "s"}, Question{Type: "lesson-polish", Kind: KindChoose, Ask: "?", Options: []string{"a"}}, host); err == nil { t.Fatal("a type outside the registry was asked") } if _, _, err := rt.Decide(context.Background(), State{Text: "s"}, Question{Type: DuplicateMatch, Kind: KindChoose, Ask: "?"}, host); err == nil { @@ -276,8 +276,10 @@ func TestTheRegistryIsClosedAndAQuestionIsChecked(t *testing.T) { t.Errorf("%s is listed but not known", ty) } } - if known("embark-lesson-ranking") { - t.Error("embark's lesson ranking is a term-overlap computation, not a judgement (iss-2610262033055327)") + // Embark's lesson ranking is a judgement routed through the adapter in + // place of its term-overlap ordering (iss-2610262033055327). + if !known(string(EmbarkLessonRanking)) || EmbarkLessonRanking != "embark-lesson-ranking" { + t.Error("embark's lesson ranking is not in the registry") } } @@ -378,7 +380,7 @@ func TestLoadRefusesAnUnlistedModelBeforeAnyCall(t *testing.T) { t.Fatalf("Load = %v, want the allowlist refusal", err) } write(`{"oracle":{` + block + `,"judgements":{"duplicate-match":"shadow:openrouter/typesafe/jev-1.13", - "remedy-read":"openrouter/typesafe/jev-1.13","lesson-ranking":"openrouter/typesafe/jev-1.13"}}}`) + "remedy-read":"openrouter/typesafe/jev-1.13","lesson-polish":"openrouter/typesafe/jev-1.13"}}}`) rt, err := Load(roots, roots.Repo) if err != nil { t.Fatal(err) @@ -386,7 +388,7 @@ func TestLoadRefusesAnUnlistedModelBeforeAnyCall(t *testing.T) { if rt.Mode(DuplicateMatch) != oracle.JudgementShadow || rt.Mode(RemedyRead) != oracle.JudgementOn || rt.Mode(MemoryCheck) != oracle.JudgementHost { t.Fatalf("modes: %s %s %s", rt.Mode(DuplicateMatch), rt.Mode(RemedyRead), rt.Mode(MemoryCheck)) } - if len(rt.Diagnostics) != 1 || !strings.Contains(rt.Diagnostics[0], "lesson-ranking") { + if len(rt.Diagnostics) != 1 || !strings.Contains(rt.Diagnostics[0], "lesson-polish") { t.Fatalf("diagnostics = %q, want the unknown type named", rt.Diagnostics) } } From 5177c296b58df9170e3f5e63b33b0e71dc4997b8 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:50:33 +0000 Subject: [PATCH 12/20] feat: turn a judgement type on only from the person's own configuration A bare / under oracle.judgements hands the type's decision to the decision adapter. That is now honoured only from the person's own configuration, the machine's ~/.abcd.noindex/config.json, the same layers a role handed to a runner is read from. The same route in a repository's .abcd/config.json is skipped with one diagnostic, and the machine's own route to the type applies in its own mode, or the host when the machine sets none, so a repository's route never turns the machine's shadow into on. A repository may still keep a type on the host or run it in shadow, where the host still decides. The refusal of a malformed judgement route in the machine's file now quotes the value as the file spells it, shadow: prefix included, and names the three forms a judgement route takes. The configuration and adapters briefs state the rule. Assisted-by: Claude:claude-opus-5-5 --- .../brief/05-internals/02-adapters.md | 5 +- .../brief/05-internals/03-configuration.md | 15 +++- internal/core/oracle/config.go | 51 ++++++++++++- internal/core/oracle/config_test.go | 14 ++-- internal/core/oracle/judgement_mode_test.go | 72 ++++++++++++++++++- 5 files changed, 144 insertions(+), 13 deletions(-) diff --git a/.abcd/development/brief/05-internals/02-adapters.md b/.abcd/development/brief/05-internals/02-adapters.md index 3bc831b796..37e44c84f4 100644 --- a/.abcd/development/brief/05-internals/02-adapters.md +++ b/.abcd/development/brief/05-internals/02-adapters.md @@ -269,7 +269,10 @@ a type moves. With `host`, or nothing set, the host decides. With recorded beside it, as a pair in the newest lab home's `pairs.jsonl`, never in the repository. A state larger than the model's 32K window is recorded as skipped with its size. With a bare `/`, the adapter decides, -and the run record names the route and the model the provider reported. An +and the run record names the route and the model the provider reported; only +the person's own `~/.abcd.noindex/config.json` turns a type on, and the same +route in a repository's file is skipped so the machine's route, or the host, +applies ([`03-configuration.md`](03-configuration.md)). An adapter that cannot answer leaves the decision to the host, with a fallback receipt saying why. The allowlist refusal of adr-2609221009491186 happens when the configuration is read, before any call. `abcd lab harvest` turns the pairs diff --git a/.abcd/development/brief/05-internals/03-configuration.md b/.abcd/development/brief/05-internals/03-configuration.md index 735c702941..860bb59a19 100644 --- a/.abcd/development/brief/05-internals/03-configuration.md +++ b/.abcd/development/brief/05-internals/03-configuration.md @@ -340,7 +340,16 @@ rather than skipped: `oracle.judgements.` value is `host`, `shadow:/` or `/`. `host`, or no route, keeps the type on the host. A repository's `host` holds against the machine's route to the same name, - since it only keeps work on the host. In **shadow** the host still decides. + since it only keeps work on the host. Only the person's own configuration + turns a type on: a bare `/` is honoured from + `~/.abcd.noindex/config.json`, the rule a role handed to a runner follows + (`runner.personal`). The same route + in a repository's `.abcd/config.json` is skipped with one diagnostic naming + the file and the route, and the machine's own route to the type applies in + its place, in its own mode, or the host when the machine sets none; so a + repository never turns a machine's shadow into on. A repository may still + set `host`, or `shadow:/`, which decides nothing. In + **shadow** the host still decides. The decision adapter is asked the same typed question over the same state beside it and decides nothing. Each pair (the host's answer, the adapter's answer and probability, the state's SHA-256 and size) is appended to @@ -393,7 +402,9 @@ rather than skipped: provider holds a key when its block names `key`, judged from the block and never by reading the credential store. A repository's route to a provider whose block names no key (a local server) is admitted and wins over the - machine's per name, and a `--route` the person types is unaffected. + machine's per name, except a judgement type's route that turns the type on, + which only the machine's file may set (above), and a `--route` the person + types is unaffected. - **A call that spends a key takes no repository settings unless the person typed its route.** A provider leg sends the connection's defaults, then the winning routing row's settings, then the `--route`'s. On a leg to a provider that holds a key and diff --git a/internal/core/oracle/config.go b/internal/core/oracle/config.go index 79ca7d912e..a00dd92f40 100644 --- a/internal/core/oracle/config.go +++ b/internal/core/oracle/config.go @@ -495,9 +495,12 @@ func (c *APIConfig) readRoutes(s *layered.Stack, key string, into map[string]Tar // The order matters: skipKeyedRoute is judged before skipUnlistedRoute, // so a keyed provider's list is never consulted on a repository's // behalf, and its route is skipped as one that would spend a key - // whatever model it names. + // whatever model it names. A judgement type's route that turns the + // type on is the person's own to set (skipTurnedOnRoute), so one + // from any other layer is skipped last, after the reasons above. for len(found) > 0 && (c.skipMalformedRoute(key, name, found[0]) || c.skipKeyedRoute(key, name, found[0]) || - c.skipUnconfiguredRoute(key, name, found) || c.skipUnlistedRoute(key, name, found[0])) { + c.skipUnconfiguredRoute(key, name, found) || c.skipUnlistedRoute(key, name, found[0]) || + c.skipTurnedOnRoute(key, name, found[0])) { found = found[1:] } if len(found) == 0 { @@ -514,6 +517,7 @@ func (c *APIConfig) readRoutes(s *layered.Stack, key string, into map[string]Tar if err != nil { return fmt.Errorf("oracle adapter: %s: %w", where, err) } + spelled := text text, mode := routeValue(key, text) if mode == JudgementHost { // The winning layer keeps the type on the host: no route, and @@ -522,7 +526,11 @@ func (c *APIConfig) readRoutes(s *layered.Stack, key string, into map[string]Tar } provider, model, ok := strings.Cut(text, "/") if !ok || provider == "" || model == "" { - return fmt.Errorf("oracle adapter: %s is %q; a route is /", where, layered.BoundKey(text)) + forms := "a route is /" + if key == judgementsKey { + forms = "a judgement type's route is host, shadow:/ or /" + } + return fmt.Errorf("oracle adapter: %s is %q; %s", where, layered.BoundKey(spelled), forms) } if _, configured := c.providers[provider]; !configured { c.Diagnostics = append(c.Diagnostics, fmt.Sprintf("oracle adapter: %s points at provider %q, which is not configured on this machine; "+ @@ -656,6 +664,43 @@ func (c *APIConfig) skipUnconfiguredRoute(key, name string, found []layered.Foun return true } +// skipTurnedOnRoute reports whether one layer's route to name is a judgement +// type's route that turns the type on, from a layer that is not the person's +// own, and says so in a diagnostic when it is. A type turned on hands its +// decision to the adapter, and only the person's own configuration may do +// that: the machine's file under their home, or their own flag for one run, +// the layers runner.personal names for a role handed to a runner (rulings RN2 +// and OC2 of 2026-10-02). The rule is restated here rather than shared, +// because runner imports this package. Skipping the route lets the next +// layer's route to the type apply, the machine's in its own mode, or the +// host when the machine sets none, so a repository never turns a machine's +// shadow into on. A repository's host and shadow are not skipped here: host +// keeps work on the host, and in shadow the host still decides. A route the +// denylist matches is left to the refusal below, which no layer softens. +func (c *APIConfig) skipTurnedOnRoute(key, name string, fd layered.Found) bool { + if key != judgementsKey || fd.Layer == layered.Machine || fd.Layer == layered.Flag { + return false + } + text, err := layered.Decode[string](fd.Raw) + if err != nil { + return false + } + target, mode := routeValue(key, text) + if mode != JudgementOn { + return false + } + _, model, _ := strings.Cut(target, "/") + if _, denied := Denied(c.denylist, model); denied { + return false + } + c.Diagnostics = append(c.Diagnostics, fmt.Sprintf("oracle adapter: %s (%s layer): %s.%s is %s, which turns the type on; "+ + "only the person's own configuration (%s) turns a judgement type on, so this route is skipped and the route %s sets "+ + "for the type, or the host, applies in its place: set it in %s, or spell it %s to run the adapter beside the host", + fd.Origin, fd.Layer, key, name, quoteUntrusted(text), layered.Config.MachineOrigin(), layered.Config.MachineOrigin(), + layered.Config.MachineOrigin(), quoteUntrusted(shadowPrefix+target))) + return true +} + // skipUnlistedRoute reports whether one layer's route to name is a route from // any layer but the machine's to a configured provider that holds no key, // naming a model that provider does not list, and says so in a diagnostic diff --git a/internal/core/oracle/config_test.go b/internal/core/oracle/config_test.go index 3f462e913b..7f75243c1f 100644 --- a/internal/core/oracle/config_test.go +++ b/internal/core/oracle/config_test.go @@ -164,7 +164,8 @@ func TestARepositoryRouteToAnUnlistedModelIsSkipped(t *testing.T) { f.machineConfig(`{"oracle":{"api":{` + localBlock + `}` + tc.machine + `}}`) other := `"roles":{"scribe":"local/qwen/qwen3-8b"}` if tc.family == rolesKey { - other = `"judgements":{"duplicate-match":"local/qwen/qwen3-8b"}` + // In shadow: only the person's own file turns a type on. + other = `"judgements":{"duplicate-match":"shadow:local/qwen/qwen3-8b"}` } f.repoConfig(`{"oracle":{` + tc.repo + `,` + other + `}}`) c, err := LoadAPI(f.roots) @@ -280,7 +281,8 @@ func TestARepositoryRouteToAKeyedProviderIsSkipped(t *testing.T) { // A keyless route beside the refused one: the skip costs nothing else. other := `"roles":{"scribe":"local/qwen/qwen3-8b"}` if tc.family == rolesKey { - other = `"judgements":{"duplicate-match":"local/qwen/qwen3-8b"}` + // In shadow: only the person's own file turns a type on. + other = `"judgements":{"duplicate-match":"shadow:local/qwen/qwen3-8b"}` } f.repoConfig(`{"oracle":{` + tc.repo + `,` + other + `}}`) c, err := LoadAPI(f.roots) @@ -479,16 +481,18 @@ func TestAMachineRouteToAKeyedProviderIsAdmitted(t *testing.T) { // TestARepositoryRouteToAKeylessProviderIsAdmitted: a provider whose block // names no key spends no key, so the repository may point at it, and its route -// wins over the machine's as every repository route does. +// wins over the machine's as every repository route does; a judgement type's +// route in shadow, since only the person's own file turns a type on +// (TestOnlyThePersonsOwnConfigurationTurnsAJudgementTypeOn). func TestARepositoryRouteToAKeylessProviderIsAdmitted(t *testing.T) { f := newFx(t) f.machineConfig(`{"oracle":{"api":{` + openrouterBlock + `,` + localBlock + `},"roles":{"scribe":"openrouter/typesafe/jev-1.13"}}}`) - f.repoConfig(`{"oracle":{"roles":{"scribe":"local/qwen/qwen3-8b"},"judgements":{"duplicate-match":"local/qwen/qwen3-8b"}}}`) + f.repoConfig(`{"oracle":{"roles":{"scribe":"local/qwen/qwen3-8b"},"judgements":{"duplicate-match":"shadow:local/qwen/qwen3-8b"}}}`) c := f.loadAPI() if tgt, ok := c.Role("scribe"); !ok || tgt.Provider != "local" || tgt.Origin != ".abcd/config.json" { t.Fatalf("role = %+v, %v", tgt, ok) } - if tgt, ok := c.Judgement("duplicate-match"); !ok || tgt.Provider != "local" { + if tgt, ok := c.Judgement("duplicate-match"); !ok || tgt.Provider != "local" || tgt.Mode != JudgementShadow { t.Fatalf("judgement = %+v, %v", tgt, ok) } } diff --git a/internal/core/oracle/judgement_mode_test.go b/internal/core/oracle/judgement_mode_test.go index 3e6dcb1ac9..056ab0530d 100644 --- a/internal/core/oracle/judgement_mode_test.go +++ b/internal/core/oracle/judgement_mode_test.go @@ -3,6 +3,8 @@ package oracle import ( "strings" "testing" + + "github.com/intentdriven/abcd/internal/abcdhome" ) // A judgement type's route carries a mode (itd-2609221009495079, @@ -97,8 +99,13 @@ func TestARepositoryHostJudgementOverridesTheMachineRoute(t *testing.T) { func TestAMalformedShadowRouteIsRefusedOnTheMachine(t *testing.T) { f := newFx(t) f.machineConfig(`{"oracle":{"api":{` + openrouterBlock + `},"judgements":{"duplicate-match":"shadow:openrouter"}}}`) - if err := f.loadAPIErr(); !strings.Contains(err.Error(), "/") { - t.Fatalf("refusal = %v", err) + // The refusal quotes the value as the file spells it, and names every + // form a judgement type's route takes. + err := f.loadAPIErr() + for _, want := range []string{`"shadow:openrouter"`, "host, shadow:/ or /"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("refusal %q does not name %q", err, want) + } } // A role takes no mode: shadow: on a role names a provider called // "shadow:openrouter", which is not a plain name and not configured. @@ -109,3 +116,64 @@ func TestAMalformedShadowRouteIsRefusedOnTheMachine(t *testing.T) { t.Fatalf("a role took the shadow mode: %+v", tgt) } } + +// A judgement type is turned over to an adapter only from the person's own +// configuration: the machine's file, or their own flag for one run, the same +// rule runner.personal applies to a role handed to a runner. A route the +// repository's file turns on is skipped with one diagnostic, and the +// machine's route to the type applies in its place, its mode with it, or the +// host when the machine sets none. The repository may still keep a type on +// the host, or run the adapter beside the host in shadow. +func TestOnlyThePersonsOwnConfigurationTurnsAJudgementTypeOn(t *testing.T) { + const repoOn = `{"oracle":{"judgements":{"eligibility-residual":"local/qwen/qwen3-8b"}}}` + for name, tc := range map[string]struct { + machine string + wantOK bool + wantMode string + }{ + "no machine route, so the host": {machine: ``}, + "the machine's shadow stays shadow": {machine: `,"judgements":{"eligibility-residual":"shadow:local/qwen/qwen3-8b"}`, + wantOK: true, wantMode: JudgementShadow}, + "the machine's own on applies": {machine: `,"judgements":{"eligibility-residual":"local/qwen/qwen3-8b"}`, + wantOK: true, wantMode: JudgementOn}, + "the machine's host stays host": {machine: `,"judgements":{"eligibility-residual":"host"}`}, + } { + t.Run(name, func(t *testing.T) { + f := newFx(t) + f.machineConfig(`{"oracle":{"api":{` + localBlock + `}` + tc.machine + `}}`) + f.repoConfig(repoOn) + c := f.loadAPI() + tgt, ok := c.Judgement("eligibility-residual") + if ok != tc.wantOK { + t.Fatalf("judgement = %+v, %v; want routed %v", tgt, ok, tc.wantOK) + } + if ok && (tgt.Mode != tc.wantMode || tgt.Origin != abcdhome.Display("config.json")) { + t.Fatalf("judgement = %+v; want the machine's route in mode %q", tgt, tc.wantMode) + } + if len(c.Diagnostics) != 1 { + t.Fatalf("diagnostics = %q, want exactly one naming the skipped route", c.Diagnostics) + } + for _, want := range []string{".abcd/config.json (repo layer)", "oracle.judgements.eligibility-residual", + "person's own configuration", "skipped", "shadow:local/qwen/qwen3-8b"} { + if !strings.Contains(c.Diagnostics[0], want) { + t.Errorf("diagnostic %q does not name %q", c.Diagnostics[0], want) + } + } + }) + } +} + +// The repository's own shadow and host stand: neither hands the decision over. +func TestARepositoryMayKeepAJudgementTypeInShadow(t *testing.T) { + f := newFx(t) + f.machineConfig(`{"oracle":{"api":{` + localBlock + `}}}`) + f.repoConfig(`{"oracle":{"judgements":{"eligibility-residual":"shadow:local/qwen/qwen3-8b"}}}`) + c := f.loadAPI() + tgt, ok := c.Judgement("eligibility-residual") + if !ok || tgt.Mode != JudgementShadow || tgt.Origin != ".abcd/config.json" { + t.Fatalf("judgement = %+v, %v; want the repository's shadow route", tgt, ok) + } + if len(c.Diagnostics) != 0 { + t.Fatalf("diagnostics = %q, want none", c.Diagnostics) + } +} From 6c7d2e552ca9df5ba3d2c9373f163789cc2a289a Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:50:35 +0000 Subject: [PATCH 13/20] fix: spell a shadow judgement route with its prefix on the providers board The text form of abcd ahoy --providers printed every judgement route as a bare /, so a shadow route read like one that turns the type on. It now prints the route as the configuration spells it. The --json form already carried the mode. Assisted-by: Claude:claude-opus-5-5 --- internal/surface/cli/ahoy_connect.go | 4 +++- internal/surface/cli/ahoy_connect_test.go | 25 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/internal/surface/cli/ahoy_connect.go b/internal/surface/cli/ahoy_connect.go index 6f2e2c1237..964b63a7e4 100644 --- a/internal/surface/cli/ahoy_connect.go +++ b/internal/surface/cli/ahoy_connect.go @@ -134,8 +134,10 @@ func runAhoyProviders(cmd *cobra.Command, cwd string, asJSON bool) error { } else { line("denylist (oracle.denylist), which refuses a model even when a provider lists it: " + strings.Join(deny, ", ")) } + // Route spells a judgement route as the configuration does, so a + // shadow route never reads as the bare form that turns a type on. for _, r := range b.Routes { - line(fmt.Sprintf("%s %s -> %s (%s)", r.Kind, r.Name, r.Target, r.Target.Origin)) + line(fmt.Sprintf("%s %s -> %s (%s)", r.Kind, r.Name, r.Target.Route(), r.Target.Origin)) } for _, d := range b.Diagnostics { line(d) diff --git a/internal/surface/cli/ahoy_connect_test.go b/internal/surface/cli/ahoy_connect_test.go index d86ba50fc6..9b821d6819 100644 --- a/internal/surface/cli/ahoy_connect_test.go +++ b/internal/surface/cli/ahoy_connect_test.go @@ -386,3 +386,28 @@ func TestAhoyConnectTakesStreamUsage(t *testing.T) { t.Fatalf("config.json:\n%s", raw) } } + +// TestAhoyProvidersSpellsAShadowJudgementRoute: the text board spells a +// judgement route as the configuration does, so a shadow route reads as +// shadow and never as the bare / that turns a type on. +func TestAhoyProvidersSpellsAShadowJudgementRoute(t *testing.T) { + hermeticEnv(t) + t.Chdir(t.TempDir()) + cfg := abcdhome.Path(os.Getenv("HOME"), "config.json") + if err := os.MkdirAll(filepath.Dir(cfg), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(cfg, []byte(`{"oracle":{"api":{"local":{"base_url":"http://localhost:11434/v1","models":["m"]}},`+ + `"judgements":{"remedy-read":"shadow:local/m","duplicate-match":"local/m"}}}`), 0o600); err != nil { + t.Fatal(err) + } + out, err := runCLIErr(t, "ahoy", "--providers") + if err != nil { + t.Fatalf("ahoy --providers: %v\n%s", err, out) + } + for _, want := range []string{"judgement remedy-read -> shadow:local/m (", "judgement duplicate-match -> local/m ("} { + if !strings.Contains(string(out), want) { + t.Errorf("the board does not say %q:\n%s", want, out) + } + } +} From f5a656b42ff1b4232ea82a3de8e006c2cc98d37c Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:50:37 +0000 Subject: [PATCH 14/20] fix: print n/a agreement for a type with nothing compared in lab harvest A judgement type whose shadow pairs were all skipped has no rate. The harvest's text output printed it as 0%; it now prints n/a, as harvest.md does. Assisted-by: Claude:claude-opus-5-5 --- internal/surface/cli/lab.go | 10 ++++++++-- internal/surface/cli/lab_surface_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/internal/surface/cli/lab.go b/internal/surface/cli/lab.go index 450a158dc0..a7776b25a0 100644 --- a/internal/surface/cli/lab.go +++ b/internal/surface/cli/lab.go @@ -361,8 +361,14 @@ func renderHarvest(w io.Writer, res lab.Harvested) { // line per judgement type, then the research note's line. if a := res.Agreement; a != nil { for _, t := range a.Types { - fmt.Fprintf(w, "agreement %s: %d/%d compared agree (%.0f%%), %d skipped\n", - termsafe.Sanitize(t.Type), t.Agreed, t.Compared, t.Rate*100, t.Skipped) + // A type with nothing compared has no rate: n/a, as harvest.md + // prints it, never 0%. + rate := "n/a" + if t.Compared > 0 { + rate = fmt.Sprintf("%.0f%%", t.Rate*100) + } + fmt.Fprintf(w, "agreement %s: %d/%d compared agree (%s), %d skipped\n", + termsafe.Sanitize(t.Type), t.Agreed, t.Compared, rate, t.Skipped) } if a.Unreadable > 0 { fmt.Fprintf(w, " %s in pairs.jsonl did not parse and are not counted\n", countOf(a.Unreadable, "line")) diff --git a/internal/surface/cli/lab_surface_test.go b/internal/surface/cli/lab_surface_test.go index ba00938662..2f46323652 100644 --- a/internal/surface/cli/lab_surface_test.go +++ b/internal/surface/cli/lab_surface_test.go @@ -191,3 +191,27 @@ func TestLabHarvestPrintsTheShadowAgreement(t *testing.T) { t.Fatalf("harvest --json leaked the home path:\n%s", out) } } + +// A type whose pairs were all skipped has nothing compared, so the harvest +// prints its agreement as n/a, as harvest.md does, never as 0%. +func TestLabHarvestPrintsNAForATypeWithNothingCompared(t *testing.T) { + _, repo := labCheckout(t) + code, out, errOut := runLab(t, "lab", "--json", "mint", "does", "the", "adapter", "agree?") + if code != 0 { + t.Fatalf("lab mint exit %d: %s%s", code, out, errOut) + } + var m struct { + ID string `json:"id"` + } + if err := json.Unmarshal([]byte(out), &m); err != nil { + t.Fatal(err) + } + if _, _, err := lab.RecordPair(repo, lab.Pair{Type: "remedy-read", Kind: "choose", StateSHA256: strings.Repeat("e", 64), + Host: "no", Target: "openrouter/typesafe/jev-1.13", Skipped: lab.SkippedWindow}); err != nil { + t.Fatal(err) + } + code, out, _ = runLab(t, "lab", "harvest", m.ID) + if code != 0 || !strings.Contains(out, "agreement remedy-read: 0/0 compared agree (n/a), 1 skipped") { + t.Fatalf("harvest (exit %d):\n%s", code, out) + } +} From 89b42f4a2414c77c2184db4ab469bd2045ffd4d9 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:50:39 +0000 Subject: [PATCH 15/20] docs: say the lab pages' shadow pairs wait on a router call The lab command page and the lab brief described shadow mode as recording pairs today. No step calls the decision adapter's router yet, so every type stays on the host and no pair is kept until that call is added; both pages now say so, as the configuration brief already did. Assisted-by: Claude:claude-opus-5-5 --- .abcd/development/brief/04-surfaces/31-lab.md | 5 ++++- commands/lab.md | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.abcd/development/brief/04-surfaces/31-lab.md b/.abcd/development/brief/04-surfaces/31-lab.md index 160bf4cc48..bb5f177b37 100644 --- a/.abcd/development/brief/04-surfaces/31-lab.md +++ b/.abcd/development/brief/04-surfaces/31-lab.md @@ -136,7 +136,10 @@ the host decides and the adapter answers beside it. Each pair is appended to `pairs.jsonl` in the newest lab home minted for the repository: the host's answer, the adapter's answer, the adapter's probability, the state's SHA-256 and size, and the model that answered, or why it did not. That is the only file the -adapter writes, and it is never written into the repository. Here the harvest +adapter writes, and it is never written into the repository. No step calls +the router yet, so every type stays on the host and no pair is kept until that +call is added ([`03-configuration.md`](../05-internals/03-configuration.md)). +Here the harvest computes something rather than only assembling prose: from `pairs.jsonl` it counts, for each judgement type, the pairs, the pairs compared (the adapter answered), the pairs that agreed and the pairs skipped (the state was larger diff --git a/commands/lab.md b/commands/lab.md index 42a206d17c..83058bd84c 100644 --- a/commands/lab.md +++ b/commands/lab.md @@ -101,6 +101,8 @@ carries `agreement`. A judgement type is put in shadow by setting `oracle.judgements.` to `shadow:/` in `~/.abcd.noindex/config.json`. The host keeps deciding, and each answer the adapter gives beside it is appended to `pairs.jsonl` in the newest lab home. +No step calls the router yet, so every type stays on the host and no pair is +kept until that call is added; until then `agreement` is absent. For each type, `agreement` gives `pairs`, `compared` (the adapter answered), `agreed`, `skipped` (the state was over the model's window, or the call failed), `rate`, and the compared and agreed counts in four probability From 613437e37ab157bd1b328a4f51ad1b275485e1c6 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:46:17 +0000 Subject: [PATCH 16/20] fix: offer a judgement type the shadow spelling after ahoy connect After connecting a provider that takes no key, ahoy connect suggested pointing a judgement type at it with the bare / in the repository's .abcd/config.json or the machine's file. A judgement type is turned on only from the person's own configuration, so the bare spelling in the repository's file is skipped. The advice now names both files for a role, offers a judgement type the shadow:/ spelling, which either file honours, and says the bare spelling turns the type on only from ~/.abcd.noindex/config.json. A provider holding a key keeps its one line naming the machine's file. Assisted-by: Claude:claude-opus-5-5 --- internal/surface/cli/ahoy_connect.go | 15 ++++++++++----- internal/surface/cli/ahoy_connect_test.go | 22 ++++++++++++++++++---- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/internal/surface/cli/ahoy_connect.go b/internal/surface/cli/ahoy_connect.go index 964b63a7e4..57cdeccc2d 100644 --- a/internal/surface/cli/ahoy_connect.go +++ b/internal/surface/cli/ahoy_connect.go @@ -265,13 +265,18 @@ func newAhoyConnectCommand(asJSON *bool) *cobra.Command { } // A route to a provider that holds a key is the machine's alone // (the ruling AA(b) of 2026-09-29, enforced by oracle.LoadAPI), so - // the repository's file is offered only for a keyless one. - where := layered.Config.MachineOrigin() + // the repository's file is offered only for a keyless one. Even + // then a judgement type is turned on only from the person's own + // configuration, so both files are offered its shadow spelling + // and the bare one is named for the machine's file alone. + machine, route := layered.Config.MachineOrigin(), res.Provider+"/"+res.Models[0] if res.KeyHome == oracle.KeyHomeNone { - where = layered.Config.RepoOrigin() + " or " + where + line(fmt.Sprintf("point a role at it with oracle.roles. = %q in %s or %s", route, layered.Config.RepoOrigin(), machine)) + line(fmt.Sprintf("put a judgement type in shadow on it with oracle.judgements. = %q in either file; the bare %q turns the type on only from %s", + "shadow:"+route, route, machine)) + } else { + line(fmt.Sprintf("point a role or a judgement type at it with oracle.roles. or oracle.judgements. = %q in %s", route, machine)) } - line(fmt.Sprintf("point a role or a judgement type at it with oracle.roles. or oracle.judgements. = %q in %s", - res.Provider+"/"+res.Models[0], where)) line(dispatchNote + ".") }) }, diff --git a/internal/surface/cli/ahoy_connect_test.go b/internal/surface/cli/ahoy_connect_test.go index 9b821d6819..d3e4acca1c 100644 --- a/internal/surface/cli/ahoy_connect_test.go +++ b/internal/surface/cli/ahoy_connect_test.go @@ -148,8 +148,10 @@ func TestAhoyConnectVerifiesThenWrites(t *testing.T) { } // TestAhoyConnectKeylessOffersBothFiles: a provider that takes no key spends -// no key, so a route to it may sit in the repository's file or the machine's, -// and the advice names both. +// no key, so a role's route to it may sit in the repository's file or the +// machine's, and the advice names both. A judgement type is offered the shadow +// spelling, which both files honour, and told that the bare spelling turns the +// type on only from the machine's file, the person's own. func TestAhoyConnectKeylessOffersBothFiles(t *testing.T) { hermeticEnv(t) t.Chdir(t.TempDir()) @@ -158,8 +160,20 @@ func TestAhoyConnectKeylessOffersBothFiles(t *testing.T) { if err != nil { t.Fatalf("ahoy connect: %v\n%s", err, out) } - if calls.Load() != 1 || !strings.Contains(string(out), `= "local/qwen/qwen3-8b" in .abcd/config.json or `+abcdhome.Display("config.json")) { - t.Fatalf("ahoy connect (%d call(s)):\n%s", calls.Load(), out) + if calls.Load() != 1 { + t.Fatalf("ahoy connect made %d call(s):\n%s", calls.Load(), out) + } + for _, want := range []string{ + `oracle.roles. = "local/qwen/qwen3-8b" in .abcd/config.json or ` + abcdhome.Display("config.json"), + `oracle.judgements. = "shadow:local/qwen/qwen3-8b" in either file`, + `turns the type on only from ` + abcdhome.Display("config.json"), + } { + if !strings.Contains(string(out), want) { + t.Errorf("ahoy connect does not say %q:\n%s", want, out) + } + } + if strings.Contains(string(out), `oracle.judgements. = "local/qwen/qwen3-8b" in .abcd/config.json`) { + t.Errorf("ahoy connect offers the repository's file a judgement route that turns the type on:\n%s", out) } } From 3ef19248e7d72ab1093bd4c5c8a1b75718079a71 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:46:20 +0000 Subject: [PATCH 17/20] docs: list a repository's turned-on judgement route among skipped routes The ahoy page lists why a route is reported as oracle_api.route_skipped. It now also names a repository's judgement route that turns a type on, which only ~/.abcd.noindex/config.json may set; the machine's route to the type, or the host, applies in its place. Assisted-by: Claude:claude-opus-5-5 --- commands/ahoy.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/commands/ahoy.md b/commands/ahoy.md index 526358ba06..59a068a340 100644 --- a/commands/ahoy.md +++ b/commands/ahoy.md @@ -1372,8 +1372,9 @@ adapter refuses as `oracle_api.config_refused`, naming the file and the key. A route the configuration read skips (a repository's route to a provider that holds a key, a repository's route that is not `/` or whose name is not a plain lower-case name, a repository's route to a model a provider -holding no key does not list, a route to a provider this machine has not -configured, or a role outside the roster) is the optional gap `oracle_api.route_skipped`, its +holding no key does not list, a repository's judgement route that turns a type +on, which only `~/.abcd.noindex/config.json` may set, a route to a provider this +machine has not configured, or a role outside the roster) is the optional gap `oracle_api.route_skipped`, its `detail` one line per skipped route; relay each line. A skipped repository route never displaces the machine's: where `~/.abcd.noindex/config.json` routes the same name, that route applies. From 5f2e03d5ac06c0c306aa30582a303ae7751044ab Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:57:54 +0000 Subject: [PATCH 18/20] fix: read a lab's shadow pairs a line at a time in lab harvest The harvest read pairs.jsonl whole under the 4 MiB bound every lab document read takes, while shadow pairs are appended to it without one, so a busy lab's harvest failed outright once the file passed the bound. It now reads the file a line at a time, each line bounded as a recorded pair is, so every pair is counted however long the lab ran; a line longer than a pair may be is counted as one that did not parse and the read goes on past it. The file is still read only as a regular file in the lab home, never through a link. Assisted-by: Claude:claude-opus-5-5 --- internal/core/lab/pairs.go | 126 ++++++++++++++++++++++---------- internal/core/lab/pairs_test.go | 58 +++++++++++++++ 2 files changed, 147 insertions(+), 37 deletions(-) diff --git a/internal/core/lab/pairs.go b/internal/core/lab/pairs.go index 526917c846..e42c50f20f 100644 --- a/internal/core/lab/pairs.go +++ b/internal/core/lab/pairs.go @@ -19,9 +19,11 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "sort" "strings" + "syscall" "github.com/intentdriven/abcd/internal/fsutil" ) @@ -159,59 +161,47 @@ var bands = []struct { }{{"0.00-0.50", 0, 0.5}, {"0.50-0.70", 0.5, 0.7}, {"0.70-0.90", 0.7, 0.9}, {"0.90-1.00", 0.9, 1.0000001}} // agreement reads the lab's pairs.jsonl; nil when the lab recorded none. +// +// The file is read a line at a time, each line bounded by maxPairBytes, and +// never whole: RecordPair appends to it for as long as a type runs in shadow, +// so a busy lab's file outgrows any bound one document read could hold, and +// the harvest still counts every pair. A line longer than a pair may be is +// counted as one that did not parse, and the read goes on past it. func (l *lab) agreement() (*Agreement, error) { - data, err := fsutil.ReadGuardedInRoot(l.root, pairsName, maxDocBytes) + f, err := l.openPairs() if errors.Is(err, os.ErrNotExist) { return nil, nil } if err != nil { return nil, fmt.Errorf("cannot read %s: %v", l.display(pairsName), redact(err, l.store.home)) } + defer f.Close() a := &Agreement{LabHome: l.store.display(l.entry.ID), Types: []TypeAgreement{}} by := map[string]*TypeAgreement{} - sc := bufio.NewScanner(bytes.NewReader(data)) - sc.Buffer(make([]byte, 0, maxPairBytes), maxPairBytes) - for sc.Scan() { - line := bytes.TrimSpace(sc.Bytes()) - if len(line) == 0 { - continue - } - var p Pair - if json.Unmarshal(line, &p) != nil || p.Type == "" || p.Host == "" { + rd := bufio.NewReaderSize(f, maxPairBytes) + for { + raw, rerr := rd.ReadSlice('\n') + if errors.Is(rerr, bufio.ErrBufferFull) { a.Unreadable++ - continue - } - a.Pairs++ - t := by[p.Type] - if t == nil { - t = &TypeAgreement{Type: p.Type, Bands: make([]Band, len(bands))} - for i, b := range bands { - t.Bands[i].Band = b.name + for errors.Is(rerr, bufio.ErrBufferFull) { + _, rerr = rd.ReadSlice('\n') + } + if rerr == io.EOF { + break + } + if rerr != nil { + return nil, fmt.Errorf("cannot read %s: %v", l.display(pairsName), redact(rerr, l.store.home)) } - by[p.Type] = t - } - t.Pairs++ - if p.Skipped != "" || p.Adapter == "" || p.Probability == nil { - t.Skipped++ continue } - t.Compared++ - agreed := p.Adapter == p.Host - if agreed { - t.Agreed++ + if rerr != nil && rerr != io.EOF { + return nil, fmt.Errorf("cannot read %s: %v", l.display(pairsName), redact(rerr, l.store.home)) } - for i, b := range bands { - if *p.Probability >= b.lo && *p.Probability < b.hi { - t.Bands[i].Compared++ - if agreed { - t.Bands[i].Agreed++ - } - } + a.count(by, bytes.TrimSpace(raw)) + if rerr == io.EOF { + break } } - if err := sc.Err(); err != nil { - return nil, fmt.Errorf("cannot read %s: %v", l.display(pairsName), err) - } names := make([]string, 0, len(by)) for n := range by { names = append(names, n) @@ -231,6 +221,68 @@ func (l *lab) agreement() (*Agreement, error) { return a, nil } +// count adds one line of pairs.jsonl to a: a pair to its type's counts and +// bands, a blank line to nothing, and anything else as unreadable. +func (a *Agreement) count(by map[string]*TypeAgreement, line []byte) { + if len(line) == 0 { + return + } + var p Pair + if json.Unmarshal(line, &p) != nil || p.Type == "" || p.Host == "" { + a.Unreadable++ + return + } + a.Pairs++ + t := by[p.Type] + if t == nil { + t = &TypeAgreement{Type: p.Type, Bands: make([]Band, len(bands))} + for i, b := range bands { + t.Bands[i].Band = b.name + } + by[p.Type] = t + } + t.Pairs++ + if p.Skipped != "" || p.Adapter == "" || p.Probability == nil { + t.Skipped++ + return + } + t.Compared++ + agreed := p.Adapter == p.Host + if agreed { + t.Agreed++ + } + for i, b := range bands { + if *p.Probability >= b.lo && *p.Probability < b.hi { + t.Bands[i].Compared++ + if agreed { + t.Bands[i].Agreed++ + } + } + } +} + +// openPairs opens the lab's pairs.jsonl to read: a regular file in the lab +// home, never a link or anything else, and the file the check vetted, not one +// put in its place since; os.ErrNotExist when the lab recorded none. +func (l *lab) openPairs() (*os.File, error) { + fi, err := l.root.Lstat(pairsName) + if err != nil { + return nil, err + } + if !fi.Mode().IsRegular() { + return nil, fsutil.ErrNotRegular + } + f, err := l.root.OpenFile(pairsName, os.O_RDONLY|syscall.O_NONBLOCK, 0) + if err != nil { + return nil, err + } + if st, err := f.Stat(); err != nil || !os.SameFile(fi, st) || !st.Mode().IsRegular() { + f.Close() + return nil, fsutil.ErrNotRegular + } + return f, nil +} + // agreementDoc renders the harvest's agreement section. func agreementDoc(a *Agreement) string { var b strings.Builder diff --git a/internal/core/lab/pairs_test.go b/internal/core/lab/pairs_test.go index 0ce09349fc..5487739068 100644 --- a/internal/core/lab/pairs_test.go +++ b/internal/core/lab/pairs_test.go @@ -148,3 +148,61 @@ func TestHarvestWithNoPairsHasNoAgreement(t *testing.T) { t.Fatal("harvest.md carries an agreement section with no pairs recorded") } } + +// TestHarvestReadsAPairsFilePastTheDocumentBound: a busy lab's pairs.jsonl +// grows past the bound one document read holds, and the harvest still reads +// every pair, a line at a time; a line longer than a pair may be is counted as +// one that did not parse, and the harvest goes on past it. +func TestHarvestReadsAPairsFilePastTheDocumentBound(t *testing.T) { + r, home := fixture(t) + notAbcd(t) + m := mint(t, r) + line, err := json.Marshal(Pair{At: "2026-10-29T06:00:00Z", Type: "eligibility-residual", Kind: "choose", + StateSHA256: strings.Repeat("d", 64), StateTokens: 300, Host: "no", Adapter: "no", Probability: prob(0.93), + Target: "openrouter/typesafe/jev-1.13", ModelReported: "typesafe/jev-1.13-0901"}) + if err != nil { + t.Fatal(err) + } + line = append(line, '\n') + n := maxDocBytes/len(line) + 1000 + var b strings.Builder + for i := range n { + if i == n/2 { + b.WriteString(strings.Repeat("x", maxPairBytes+10) + "\n") + } + b.Write(line) + } + path := filepath.Join(labDir(home, m), "pairs.jsonl") + if err := os.WriteFile(path, []byte(b.String()), 0o600); err != nil { + t.Fatal(err) + } + if fi, err := os.Stat(path); err != nil || fi.Size() <= maxDocBytes { + t.Fatalf("the fixture is %v bytes, %v; want more than %d", fi, err, maxDocBytes) + } + res, err := Harvest(r.Root(), m.ID) + if err != nil { + t.Fatalf("Harvest over a pairs file past the bound: %v", err) + } + a := res.Agreement + if a == nil || a.Pairs != n || a.Unreadable != 1 || len(a.Types) != 1 || a.Types[0].Agreed != n { + t.Fatalf("agreement = %+v, want %d pairs, all agreed, and one unreadable line", a, n) + } +} + +// TestHarvestRefusesALinkedPairsFile: the harvest reads pairs.jsonl only as a +// regular file in the lab home, never through a link to one elsewhere. +func TestHarvestRefusesALinkedPairsFile(t *testing.T) { + r, home := fixture(t) + notAbcd(t) + m := mint(t, r) + elsewhere := filepath.Join(t.TempDir(), "pairs.jsonl") + if err := os.WriteFile(elsewhere, []byte(`{"type":"remedy-read","kind":"choose","host":"yes"}`+"\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(elsewhere, filepath.Join(labDir(home, m), "pairs.jsonl")); err != nil { + t.Fatal(err) + } + if _, err := Harvest(r.Root(), m.ID); err == nil || !strings.Contains(err.Error(), "pairs.jsonl") { + t.Fatalf("Harvest through a linked pairs file = %v, want a refusal naming it", err) + } +} From 9ef569a108defbbceccae852cef4e7904ca63011 Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Fri, 30 Oct 2026 01:54:58 +0000 Subject: [PATCH 19/20] feat: route the drain's judgement through the decision adapter The drain's judgement before a lane opens is the judgement type eligibility-residual, and the drain's move now asks the router for it. On the host, the default, the move is as it was. In shadow, once the host's or the routed judge's answer is taken, the adapter is asked the same question over the same record and the pair is recorded in the newest lab home; a state over the adapter's window is recorded as skipped with its size. Shadow decides nothing, and its call is bounded by its own short timeout. Turned on (only from the person's own configuration), the adapter is asked once the host's request is written, before any runner the judge is routed to; its answer is taken as a routed judge's is, and the drain's record names the route decision-adapter /. An adapter that cannot answer leaves the host's request standing with a fallback receipt in the drain's state, and the request is marked as asked, so the polls that follow while it stands ask the adapter nothing again. A drain stopped while the adapter is asked refuses the move and records no fallback. Assisted-by: Claude:claude-opus-5-5 --- internal/core/implement/loop/drain.go | 33 ++- .../implement/loop/drain_adapter_wire_test.go | 214 ++++++++++++++++++ internal/core/implement/loop/drain_decide.go | 68 +++++- .../core/implement/loop/drain_decide_test.go | 13 +- 4 files changed, 308 insertions(+), 20 deletions(-) create mode 100644 internal/core/implement/loop/drain_adapter_wire_test.go diff --git a/internal/core/implement/loop/drain.go b/internal/core/implement/loop/drain.go index d3dad30ae5..dc588c6cec 100644 --- a/internal/core/implement/loop/drain.go +++ b/internal/core/implement/loop/drain.go @@ -634,6 +634,10 @@ func drainMove(repoRoot string, o Options, d DrainOptions, flags paceFlags, rule st.Judging = nil res.Judged = &j pruneLapse(&st, j.Issue) + // With eligibility-residual in shadow, the decision adapter is asked + // beside the answer just taken and the pair is recorded in the newest + // lab home; it changes nothing the move does (drain_decide.go). + shadowDrainJudgement(d.Context, repoRoot, o, j, plan) return nil } if d.Judgement != "" { @@ -1026,22 +1030,31 @@ func drainMove(repoRoot string, o Options, d DrainOptions, flags paceFlags, rule judging.Hold = judgeHold st.Judging = &judging pruneLapse(&st, v.ID) - // A judge routed to a runner answers in this move, taken as the - // host's answer is; one that does not answer leaves the host's - // request standing, and is not started again for it - // (drain_judge.go). + // The decision adapter, when eligibility-residual is turned on, + // and then a judge routed to a runner answer in this move, taken + // as the host's answer is; one that does not answer leaves the + // host's request standing, and is not asked again for it + // (drain_decide.go, drain_judge.go). var taken *DrainJudgement clause := "" if judging.RoutedAsked { - clause = judgeAskedClause(v.ID) + clause = adapterAskedClause(repoRoot, o, d, v.ID) } else { - var started bool - if taken, clause, started, err = routedJudgement(repoRoot, iss, v, judging, plan, &st, o, d, now); err != nil { + var adapterAsked, started bool + if taken, clause, adapterAsked, err = adapterJudgement(d.Context, repoRoot, o, judging, plan, &st, now); err != nil { return refuseMoved(err) } - // Marked as asked only once the runner was started: a - // judge that never started is asked again next move. - if taken == nil && started { + if taken == nil { + var routed string + if taken, routed, started, err = routedJudgement(repoRoot, iss, v, judging, plan, &st, o, d, now); err != nil { + return refuseMoved(err) + } + clause = joinNext(clause, routed) + } + // Marked as asked only once the adapter was asked or the + // runner was started: a judge that never started is asked + // again next move. + if taken == nil && (adapterAsked || started) { st.Judging.RoutedAsked = true } } diff --git a/internal/core/implement/loop/drain_adapter_wire_test.go b/internal/core/implement/loop/drain_adapter_wire_test.go new file mode 100644 index 0000000000..2bae9c801b --- /dev/null +++ b/internal/core/implement/loop/drain_adapter_wire_test.go @@ -0,0 +1,214 @@ +package loop + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/intentdriven/abcd/internal/adapter/openaiapi" + "github.com/intentdriven/abcd/internal/core/lab" + "github.com/intentdriven/abcd/internal/core/oracle" + "github.com/intentdriven/abcd/internal/core/runner" +) + +// The drain's judgement through the decision adapter, as a drain move runs it +// (itd-2609221009495079 criteria 1, 3 and 5): the type eligibility-residual on +// the host leaves the drain as it was; in shadow the host's answer is taken +// and the adapter's is recorded beside it, or skipped with the state's size +// when the state is over the window; turned on, the adapter's answer is taken +// in the move that asks it, the run record naming the route, and an adapter +// that cannot answer leaves the host's request standing with a fallback +// receipt in the drain's state, asked once per request. + +// drainStateOf reads the drain's state as a move left it. +func drainStateOf(t *testing.T, root string) DrainState { + t.Helper() + raw, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(DrainStateRel))) + if err != nil { + t.Fatal(err) + } + var st DrainState + if err := json.Unmarshal(raw, &st); err != nil { + t.Fatal(err) + } + return st +} + +// TestADrainOnTheHostAsksNoAdapter: with eligibility-residual on the host, +// the move writes the host's request as it always did, the host's answer +// opens the lane, and the adapter is never asked and no pair is kept. +func TestADrainOnTheHostAsksNoAdapter(t *testing.T) { + repo := issueRepo(t) + adapter, got := &drainFakeJudge{value: HandBackTrustRule, p: 0.9}, []lab.Pair{} + withDrainRouter(t, oracle.JudgementHost, adapter, &got) + res, err := Drain(repo.Root(), Options{}, DrainOptions{}) + if err != nil { + t.Fatal(err) + } + if res.Judging == nil || res.Judged != nil || !strings.Contains(res.Next, "abcd drain --judgement") || strings.Contains(res.Next, drainAdapterRoute) { + t.Fatalf("the host's request is written and awaited: %+v", res) + } + res = judge(t, repo, Options{}, res, JudgementNo, "", "a renderer guard") + if res.Judged == nil || res.Judged.Route != nil || res.Start == nil { + t.Fatalf("the host's no is taken and opens the lane: %+v", res) + } + if adapter.calls != 0 || len(got) != 0 || len(drainStateOf(t, repo.Root()).JudgeFallbacks) != 0 { + t.Fatalf("on the host the adapter is asked %d times, %d pairs are kept", adapter.calls, len(got)) + } +} + +// TestADrainInShadowRecordsThePairAndKeepsTheHostsAnswer is criterion 1 in +// the drain: the host's answer decides, and once it is taken the adapter is +// asked the same question and the pair is recorded; the adapter's different +// answer changes nothing. +func TestADrainInShadowRecordsThePairAndKeepsTheHostsAnswer(t *testing.T) { + repo := issueRepo(t) + adapter, got := &drainFakeJudge{value: HandBackTrustRule, p: 0.8}, []lab.Pair{} + withDrainRouter(t, oracle.JudgementShadow, adapter, &got) + res, err := Drain(repo.Root(), Options{}, DrainOptions{}) + if err != nil { + t.Fatal(err) + } + if res.Judging == nil || adapter.calls != 0 { + t.Fatalf("shadow asks the adapter nothing before the host answers: %+v, calls %d", res, adapter.calls) + } + res = judge(t, repo, Options{}, res, JudgementNo, "", "a renderer guard") + if res.Judged == nil || res.Judged.Answer != JudgementNo || res.Judged.Route != nil || res.Start == nil { + t.Fatalf("the host's no is taken and opens the lane, whatever the adapter said: %+v", res) + } + if adapter.calls != 1 || len(got) != 1 { + t.Fatalf("the adapter is asked once beside the host and one pair kept: calls %d, pairs %+v", adapter.calls, got) + } + p := got[0] + if p.Type != "eligibility-residual" || p.Host != JudgementNo || p.Adapter != HandBackTrustRule || p.Probability == nil || + *p.Probability != 0.8 || len(p.StateSHA256) != 64 || p.Skipped != "" { + t.Fatalf("pair = %+v", p) + } + if st := drainStateOf(t, repo.Root()); len(st.JudgeFallbacks) != 0 { + t.Fatalf("shadow keeps no fallback in the drain's state: %+v", st.JudgeFallbacks) + } +} + +// TestADrainInShadowSkipsAStateOverTheWindow is criterion 5 in the drain: a +// state larger than the adapter's window is not sent; the pair is recorded as +// skipped with the state's size, and the host's answer decides alone. +func TestADrainInShadowSkipsAStateOverTheWindow(t *testing.T) { + repo := issueRepo(t) + adapter, got := &drainFakeJudge{value: JudgementNo, p: 0.9}, []lab.Pair{} + withDrainRouterWindow(t, oracle.JudgementShadow, 8, adapter, &got) + res, err := Drain(repo.Root(), Options{}, DrainOptions{}) + if err != nil { + t.Fatal(err) + } + res = judge(t, repo, Options{}, res, JudgementNo, "", "a renderer guard") + if res.Judged == nil || res.Start == nil { + t.Fatalf("the host's no opens the lane: %+v", res) + } + if adapter.calls != 0 || len(got) != 1 { + t.Fatalf("a state over the window is not sent: calls %d, pairs %+v", adapter.calls, got) + } + if p := got[0]; p.Skipped != lab.SkippedWindow || p.StateTokens <= 8 || p.Host != JudgementNo || + !strings.Contains(p.Detail, "tokens") || p.Adapter != "" { + t.Fatalf("pair = %+v, want a window skip with the size", p) + } +} + +// TestADrainTurnedOnTakesTheAdaptersAnswerAndNamesTheRoute is criterion 3's +// first half in the drain: the adapter's no is taken in the move that asks +// it, the lane opens, and the drain's record names the adapter's route. +func TestADrainTurnedOnTakesTheAdaptersAnswerAndNamesTheRoute(t *testing.T) { + repo := issueRepo(t) + adapter, got := &drainFakeJudge{value: JudgementNo, p: 0.97}, []lab.Pair{} + withDrainRouter(t, oracle.JudgementOn, adapter, &got) + res, err := Drain(repo.Root(), Options{}, DrainOptions{}) + if err != nil { + t.Fatal(err) + } + if res.Judging != nil || res.Judged == nil || res.Judged.Answer != JudgementNo || !res.Judged.Applied || res.Start == nil { + t.Fatalf("the adapter's no is taken and opens the lane in the same move: %+v", res) + } + want := drainAdapterRoute + " openrouter/typesafe/jev-1.13" + st := drainStateOf(t, repo.Root()) + if len(st.Judgements) != 1 { + t.Fatalf("judgements = %+v", st.Judgements) + } + if r := st.Judgements[0].Route; r == nil || r.Asked != want || r.Ran != want || r.Model != "typesafe/jev-1.13-0901" { + t.Fatalf("the drain's record names the route: %+v", r) + } + if adapter.calls != 1 || len(got) != 0 || len(st.JudgeFallbacks) != 0 { + t.Fatalf("calls %d, pairs %d, fallbacks %+v", adapter.calls, len(got), st.JudgeFallbacks) + } +} + +// TestADrainTurnedOnFallsBackToTheHostWithAReceiptOnce is criterion 3's +// second half in the drain: an adapter that cannot answer leaves the host's +// request standing, the fallback receipt kept in the drain's state; the polls +// that follow while the request stands ask it nothing again, and the host's +// answer is taken as it always was. +func TestADrainTurnedOnFallsBackToTheHostWithAReceiptOnce(t *testing.T) { + repo := issueRepo(t) + adapter, got := &drainFakeJudge{err: openaiapi.ErrUnreachable}, []lab.Pair{} + withDrainRouter(t, oracle.JudgementOn, adapter, &got) + res, err := Drain(repo.Root(), Options{}, DrainOptions{}) + if err != nil { + t.Fatal(err) + } + if res.Judging == nil || res.Judged != nil || res.Start != nil { + t.Fatalf("the host's request stands: %+v", res) + } + if !strings.Contains(res.Next, "abcd drain --judgement") || !strings.Contains(res.Next, "the decision adapter did not judge "+eligibleIssue) { + t.Fatalf("next names the fallback and the host's answer: %s", res.Next) + } + fbs := drainStateOf(t, repo.Root()).JudgeFallbacks + if len(fbs) != 1 || fbs[0].Role != RoleDrainJudge || fbs[0].Ran != runner.Host || + fbs[0].Asked != drainAdapterRoute+" openrouter/typesafe/jev-1.13" || !strings.Contains(fbs[0].Detail, "the host's request stands") { + t.Fatalf("the fallback receipt is kept in the drain's state: %+v", fbs) + } + for range 2 { + poll, err := Drain(repo.Root(), Options{}, DrainOptions{}) + if err != nil { + t.Fatal(err) + } + if poll.Judging == nil || poll.Judged != nil || !strings.Contains(poll.Next, "the decision adapter was already asked over "+eligibleIssue) { + t.Fatalf("the host's request still stands, and the adapter is not asked again: %+v", poll) + } + res = poll + } + if adapter.calls != 1 || len(drainStateOf(t, repo.Root()).JudgeFallbacks) != 1 { + t.Fatalf("the adapter is asked once per request: %d calls, fallbacks %+v", adapter.calls, drainStateOf(t, repo.Root()).JudgeFallbacks) + } + res = judge(t, repo, Options{}, res, JudgementNo, "", "a renderer guard") + if res.Judged == nil || res.Judged.Route != nil || res.Start == nil { + t.Fatalf("the host's answer is taken and opens the lane: %+v", res) + } +} + +// TestADrainStoppedWhileTheAdapterJudgesTakesNoAnswer: a drain stopped while +// the decision adapter is asked refuses the move, records no fallback for the +// stop, and leaves the request unasked, so the next move asks again. +func TestADrainStoppedWhileTheAdapterJudgesTakesNoAnswer(t *testing.T) { + repo := issueRepo(t) + adapter, got := &drainFakeJudge{err: context.Canceled}, []lab.Pair{} + withDrainRouter(t, oracle.JudgementOn, adapter, &got) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := Drain(repo.Root(), Options{}, DrainOptions{Context: ctx}); err == nil || !strings.Contains(err.Error(), "stopped while the decision adapter judged") { + t.Fatalf("a stopped drain refuses the move: %v", err) + } + // A first move refused writes no state at all; any it wrote keeps no + // fallback and leaves the request unasked. + if _, err := os.Stat(filepath.Join(repo.Root(), filepath.FromSlash(DrainStateRel))); err == nil { + st := drainStateOf(t, repo.Root()) + if len(st.JudgeFallbacks) != 0 || (st.Judging != nil && st.Judging.RoutedAsked) { + t.Fatalf("the stop is no fallback and the request is asked again: %+v", st) + } + } + adapter.err, adapter.value, adapter.p = nil, JudgementNo, 0.9 + res, err := Drain(repo.Root(), Options{}, DrainOptions{}) + if err != nil || res.Judged == nil || res.Judged.Route == nil || adapter.calls != 2 { + t.Fatalf("the next move asks the adapter again and takes its answer: %+v, %v, calls %d", res, err, adapter.calls) + } +} diff --git a/internal/core/implement/loop/drain_decide.go b/internal/core/implement/loop/drain_decide.go index ed0d33b1b1..2d4dd2fd91 100644 --- a/internal/core/implement/loop/drain_decide.go +++ b/internal/core/implement/loop/drain_decide.go @@ -11,17 +11,22 @@ package loop // - Configured for shadow, the host (or the routed runner) decides exactly // as it does today; once its answer is taken, the adapter is asked the // same question over the same record and the pair is recorded in the -// newest lab home (shadowDrainJudgement). Shadow never changes, delays or -// stops the drain. +// newest lab home (shadowDrainJudgement). Shadow never changes or stops +// the drain, and its one call, made after the answer is taken, is bounded +// by drainShadowTimeout. // - Turned on, the adapter is asked once the host's request is written -// (decideDrainJudgement): its answer is taken as a routed judge's is, +// (decideDrainJudgement, through adapterJudgement), before any runner the +// drain's judge is routed to: its answer is taken as a routed judge's is, // under the same disposition rules (judgeAnswer), and the judgement's // Route names the adapter and the model that answered. An adapter that // cannot answer leaves the host's request standing, with a fallback -// receipt kept beside the routed judge's. +// receipt kept beside the routed judge's, and the request is marked as +// asked, so the polls that follow ask it nothing again. // // Both functions read the configuration themselves (loadDrainRouter), so the -// drain's call sites pass only what they already hold. +// drain's call sites pass only what they already hold. On the host, the +// default, neither asks anything, and the drain moves as it did before the +// adapter existed. import ( "context" @@ -49,6 +54,11 @@ const drainAdapterRoute = "decision-adapter" // takes about a second, so this is for a provider that hangs. const drainAdapterTimeout = 2 * time.Minute +// drainShadowTimeout bounds the adapter's call in shadow, where its answer +// decides nothing: a provider slower than this costs the pair (recorded as +// failed), never the move's time. +const drainShadowTimeout = 20 * time.Second + // loadDrainRouter reads the decision adapter's configuration for the checkout // at repoRoot. Tests replace it. var loadDrainRouter = func(repoRoot string, o Options) (*decision.Router, error) { @@ -92,7 +102,8 @@ func drainValue(j DrainJudgement) string { // the pair. It returns the receipt, nil when the type is not in shadow, the // issue has left the open ledger, or the configuration cannot be read (the // drain has its answer; shadow never stops it). The drain's take point calls -// it before applying j. +// it once j is applied and recorded, so an answer the move could not take is +// never paired. func shadowDrainJudgement(ctx context.Context, repoRoot string, o Options, j DrainJudgement, plan capture.DrainPlan) *decision.Receipt { iss, ok := plan.OpenIssue(j.Issue) if !ok || drainValue(j) == "" { @@ -102,7 +113,7 @@ func shadowDrainJudgement(ctx context.Context, repoRoot string, o Options, j Dra if err != nil || r.Mode(drainJudgementType) != oracle.JudgementShadow { return nil } - ctx, cancel := context.WithTimeout(ctxOr(ctx), drainAdapterTimeout) + ctx, cancel := context.WithTimeout(ctxOr(ctx), drainShadowTimeout) defer cancel() _, rc, err := r.Decide(ctx, drainState(iss), drainQuestion(), decision.Answered(drainValue(j))) if err != nil { @@ -138,7 +149,7 @@ func decideDrainJudgement(ctx context.Context, repoRoot string, o Options, judgi asked := drainAdapterRoute + " " + rc.Target if errors.Is(err, decision.ErrHostPending) { return nil, &runner.FallbackReceipt{At: now, Role: RoleDrainJudge, Asked: asked, Reason: runner.ReasonFailed, - Detail: rc.Fallback + "; the host's request stands", Ran: runner.Host}, nil + Detail: strings.TrimSuffix(rc.Fallback, "the host decided") + "the host's request stands", Ran: runner.Host}, nil } if err != nil { return nil, nil, err @@ -157,6 +168,47 @@ func decideDrainJudgement(ctx context.Context, repoRoot string, o Options, judgi return &taken, nil, nil } +// adapterJudgement is the drain's move over the request it awaits, judging, +// when eligibility-residual is turned on: the adapter's answer, taken as a +// routed judge's is, or nil with the clause the move's next step adds and the +// fallback receipt kept in st, the host's request then standing. asked +// reports whether the adapter was asked and did not answer, so the request is +// marked as asked (DrainJudging.RoutedAsked) and the polls that follow while +// the host's request stands ask it nothing again. Not turned on, it asks +// nothing and returns nil, "", false. A drain stopped while the adapter was +// asked is no fallback: the move refuses, and the next one asks again. +func adapterJudgement(ctx context.Context, repoRoot string, o Options, judging DrainJudging, plan capture.DrainPlan, + st *DrainState, now time.Time) (taken *DrainJudgement, clause string, asked bool, err error) { + ctx = ctxOr(ctx) + taken, fb, err := decideDrainJudgement(ctx, repoRoot, o, judging, plan, now) + if err != nil { + return nil, "", false, err + } + if fb == nil { + return taken, "", false, nil + } + if ctx.Err() != nil { + return nil, "", false, refuse(StageDrain, "", "", "the drain was stopped while the decision adapter judged "+judging.Issue+ + "; the move took no answer", "run `abcd drain` again, which asks the decision adapter again") + } + keepJudgeFallback(st, *fb) + return nil, "the decision adapter did not judge " + judging.Issue + ", recorded as a fallback: " + fb.Detail, true, nil +} + +// adapterAskedClause is the move's clause for a request marked as asked +// (DrainJudging.RoutedAsked) when the drain's judge is not routed to a +// runner and eligibility-residual is turned on: the decision adapter was the +// one asked, in an earlier move, and is not asked again. Otherwise it is the +// routed runner's clause (judgeAskedClause). +func adapterAskedClause(repoRoot string, o Options, d DrainOptions, issue string) string { + if !judgeRouted(o, d) { + if r, err := loadDrainRouter(repoRoot, o); err == nil && r.Mode(drainJudgementType) == oracle.JudgementOn { + return "the decision adapter was already asked over " + issue + " for this request and did not answer, so it is not asked again and the request is the host's" + } + } + return judgeAskedClause(issue) +} + // ctxOr is ctx, or the background context when it is nil. func ctxOr(ctx context.Context) context.Context { if ctx == nil { diff --git a/internal/core/implement/loop/drain_decide_test.go b/internal/core/implement/loop/drain_decide_test.go index 19140b23a3..4e538eee90 100644 --- a/internal/core/implement/loop/drain_decide_test.go +++ b/internal/core/implement/loop/drain_decide_test.go @@ -42,6 +42,13 @@ func (f *drainFakeJudge) Judge(context.Context, decision.State, decision.Questio // withDrainRouter makes the drain judge with eligibility-residual in mode, // adapter answering, and pairs kept in got. func withDrainRouter(t *testing.T, mode string, adapter *drainFakeJudge, got *[]lab.Pair) { + t.Helper() + withDrainRouterWindow(t, mode, 0, adapter, got) +} + +// withDrainRouterWindow is withDrainRouter with the adapter's window in +// tokens; the default window when zero. +func withDrainRouterWindow(t *testing.T, mode string, window int, adapter *drainFakeJudge, got *[]lab.Pair) { t.Helper() routes := map[decision.Type]oracle.Target{} if mode != oracle.JudgementHost { @@ -49,10 +56,12 @@ func withDrainRouter(t *testing.T, mode string, adapter *drainFakeJudge, got *[] } prev := loadDrainRouter loadDrainRouter = func(string, Options) (*decision.Router, error) { - return decision.NewForTest(routes, func(oracle.Target) decision.Judge { return adapter }, func(p lab.Pair) (string, string, error) { + r := decision.NewForTest(routes, func(oracle.Target) decision.Judge { return adapter }, func(p lab.Pair) (string, string, error) { *got = append(*got, p) return "lab-261029000000-abcdef0", abcdhome.Display("lab/x/lab-261029000000-abcdef0/pairs.jsonl"), nil - }), nil + }) + r.Window = window + return r, nil } t.Cleanup(func() { loadDrainRouter = prev }) } From 6d1e887749b8f78de8fe0638c9b626268b5c697d Mon Sep 17 00:00:00 2001 From: REPPL <77722411+REPPL@users.noreply.github.com> Date: Fri, 30 Oct 2026 01:54:59 +0000 Subject: [PATCH 20/20] docs: say the drain's judgement goes through the decision adapter The lab page, the lab and configuration chapters, the adapters chapter and the package map said no step calls the router yet. The drain's judgement before a lane opens now does, in each mode; every other judgement type still stays on the host whatever its route says. Assisted-by: Claude:claude-opus-5-5 --- .abcd/development/brief/04-surfaces/31-lab.md | 8 ++++-- .../brief/05-internals/02-adapters.md | 10 +++---- .../brief/05-internals/03-configuration.md | 28 ++++++++++--------- commands/lab.md | 6 ++-- internal/README.md | 4 ++- 5 files changed, 32 insertions(+), 24 deletions(-) diff --git a/.abcd/development/brief/04-surfaces/31-lab.md b/.abcd/development/brief/04-surfaces/31-lab.md index bb5f177b37..6068c77a9b 100644 --- a/.abcd/development/brief/04-surfaces/31-lab.md +++ b/.abcd/development/brief/04-surfaces/31-lab.md @@ -136,9 +136,11 @@ the host decides and the adapter answers beside it. Each pair is appended to `pairs.jsonl` in the newest lab home minted for the repository: the host's answer, the adapter's answer, the adapter's probability, the state's SHA-256 and size, and the model that answered, or why it did not. That is the only file the -adapter writes, and it is never written into the repository. No step calls -the router yet, so every type stays on the host and no pair is kept until that -call is added ([`03-configuration.md`](../05-internals/03-configuration.md)). +adapter writes, and it is never written into the repository. The drain's +judgement before a lane opens (`eligibility-residual`) is today the one step +that goes through the router, so it is the one type whose pairs are kept; +every other type stays on the host whatever its route says +([`03-configuration.md`](../05-internals/03-configuration.md)). Here the harvest computes something rather than only assembling prose: from `pairs.jsonl` it counts, for each judgement type, the pairs, the pairs compared (the adapter diff --git a/.abcd/development/brief/05-internals/02-adapters.md b/.abcd/development/brief/05-internals/02-adapters.md index 37e44c84f4..fb056b9f1b 100644 --- a/.abcd/development/brief/05-internals/02-adapters.md +++ b/.abcd/development/brief/05-internals/02-adapters.md @@ -280,11 +280,11 @@ into agreement per type, with counts and probability bands, and the research note filed from it is the evidence a ruling turns a type on with. The first candidate is `typesafe/jev-1.13` on OpenRouter's list. The live paid check against it is owed to a person (iss-2609291925398978). The drain's judgement -(`eligibility-residual`) has its routing built but is not yet called from the -drain's move, and no other type's step calls the router yet, so until those -calls are added every judgement stays on the host. The tests drive the -router, the pairs file, the harvest and the drain's routing with a fake -adapter and a temporary home. +(`eligibility-residual`) is the one type whose step calls the router: the +drain's move asks it in each mode. No other type's step calls the router yet, +so every other judgement stays on the host whatever its route says. The tests +drive the router, the pairs file, the harvest and the drain's moves in each +mode with a fake adapter and a temporary home. ### Reviewer routes — the command-line runner, and RepoPrompt over MCP diff --git a/.abcd/development/brief/05-internals/03-configuration.md b/.abcd/development/brief/05-internals/03-configuration.md index 860bb59a19..0da0b52593 100644 --- a/.abcd/development/brief/05-internals/03-configuration.md +++ b/.abcd/development/brief/05-internals/03-configuration.md @@ -370,19 +370,21 @@ rather than skipped: `source-selection`, `memory-check`, `ledger-category`, `ledger-severity`, `prepass-overlap`, `present-tense-advisory`, `embark-lesson-ranking`. A route naming anything else is - a diagnostic and is skipped. The drain's judgement before a lane opens is - meant to run as `eligibility-residual`, a choice of `no`, `user-visible` or - `trust-rule`. Its routing is built (`loop/drain_decide.go`). In shadow, the - routing records a pair once the host's or the routed judge's answer is - taken. Turned on, it asks the adapter once the host's request is written, - and takes the answer the way a routed judge's is taken, with the route named - `decision-adapter /`. If the adapter cannot answer, the - host's request stands, with a fallback receipt beside the routed judge's. - Nothing in the drain's move calls this routing yet, so until that call is - added the drain's judgement stays on the host whatever this route says. No - other type's step calls the router yet either. Those judgements are made in - the host's command pages, which hand nothing back to the binary, so a route - to one of them takes effect only once its step calls the router. `abcd lab harvest` reports the agreement per type, and + a diagnostic and is skipped. The drain's judgement before a lane opens runs + as `eligibility-residual`, a choice of `no`, `user-visible` or + `trust-rule`, and the drain's move calls the router for it + (`loop/drain_decide.go`). On the host, the default, the drain moves as it + always did. In shadow, the move records a pair once the host's or the + routed judge's answer is taken, and changes nothing else. Turned on, the + move asks the adapter once the host's request is written, before any runner + the drain's judge is routed to, and takes the answer the way a routed + judge's is taken, with the route named `decision-adapter /` + in the drain's record. If the adapter cannot answer, the host's request + stands, with a fallback receipt in the drain's state beside the routed + judge's, and the adapter is not asked again for that request. No other + type's step calls the router yet. Those judgements are made in the host's + command pages, which hand nothing back to the binary, so they stay on the + host whatever their route says until their step calls the router. `abcd lab harvest` reports the agreement per type, and the research note filed from it is the evidence a ruling turns a type on with. - **A route to a provider that holds a key sits on the machine alone.** Only a diff --git a/commands/lab.md b/commands/lab.md index 83058bd84c..5485d24479 100644 --- a/commands/lab.md +++ b/commands/lab.md @@ -101,8 +101,10 @@ carries `agreement`. A judgement type is put in shadow by setting `oracle.judgements.` to `shadow:/` in `~/.abcd.noindex/config.json`. The host keeps deciding, and each answer the adapter gives beside it is appended to `pairs.jsonl` in the newest lab home. -No step calls the router yet, so every type stays on the host and no pair is -kept until that call is added; until then `agreement` is absent. +Today the drain's judgement before a lane opens (`eligibility-residual`) is +the one step that goes through the router, so it is the one type whose pairs +are kept; every other type stays on the host whatever its route says, and a +lab with no pairs has no `agreement`. For each type, `agreement` gives `pairs`, `compared` (the adapter answered), `agreed`, `skipped` (the state was over the model's window, or the call failed), `rate`, and the compared and agreed counts in four probability diff --git a/internal/README.md b/internal/README.md index dff48ed46f..7bc54c69d1 100644 --- a/internal/README.md +++ b/internal/README.md @@ -248,7 +248,9 @@ plugin surface, and a future MCP server share one engine. provider's allowlist. In shadow the host decides and the pair is recorded in the newest lab home's `pairs.jsonl` (`core/lab`). Turned on, the adapter decides, and an adapter that cannot answer leaves the decision to the host - with a receipt. + with a receipt. The drain's judgement before a lane opens + (`eligibility-residual`, `core/implement/loop`) is the one step that calls + it; every other type stays on the host. - **`core/decisions/`** — the decision records' folder, `.abcd/work/decisions/`: one `dcn--.md` file per decision (adr-2610110925465593). `Record` is its one writer, behind `abcd decide ""`: it mints through