diff --git a/a2aremote/go.mod b/a2aremote/go.mod index 25da01cd..e71196f8 100644 --- a/a2aremote/go.mod +++ b/a2aremote/go.mod @@ -7,8 +7,8 @@ replace github.com/xraph/cortex => ../ require ( github.com/stretchr/testify v1.12.1 github.com/xraph/cortex v1.6.1 - github.com/xraph/grove v1.10.2 - github.com/xraph/grove/drivers/sqlitedriver v1.10.2 + github.com/xraph/grove v1.10.3 + github.com/xraph/grove/drivers/sqlitedriver v1.10.3 ) require ( diff --git a/a2aremote/go.sum b/a2aremote/go.sum index 1fa99730..52e43405 100644 --- a/a2aremote/go.sum +++ b/a2aremote/go.sum @@ -31,10 +31,10 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/xraph/shield v1.10.2 h1:LJlnTeXvDov+GBPhOGsnyoiH/1Zsx5gaOPfNfSAGoxU= github.com/xraph/shield v1.10.2/go.mod h1:xkWUxiFghxQV74UVNfDsSz/mCjQ18mpt4W4kxGiBqJU= github.com/xraph/vessel v1.0.5 h1:4In3UmIJzOrym5+FXE/uvm1jR+gofUqVmEjKjjgE3gw= diff --git a/a2aremote/grpcbind/go.sum b/a2aremote/grpcbind/go.sum index c8b76f21..58e5e60a 100644 --- a/a2aremote/grpcbind/go.sum +++ b/a2aremote/grpcbind/go.sum @@ -29,10 +29,10 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/xraph/shield v1.10.2 h1:LJlnTeXvDov+GBPhOGsnyoiH/1Zsx5gaOPfNfSAGoxU= github.com/xraph/shield v1.10.2/go.mod h1:xkWUxiFghxQV74UVNfDsSz/mCjQ18mpt4W4kxGiBqJU= github.com/xraph/vessel v1.0.5 h1:4In3UmIJzOrym5+FXE/uvm1jR+gofUqVmEjKjjgE3gw= diff --git a/api/go.mod b/api/go.mod index ac6f9bf9..a42d4b40 100644 --- a/api/go.mod +++ b/api/go.mod @@ -9,8 +9,8 @@ require ( github.com/xraph/cortex v1.8.1-0.20261025225257-d0eb82f6ed68 github.com/xraph/cortex/optimization v0.0.0-20261026160546-1b6f6bc9f367 github.com/xraph/forge v1.13.6 - github.com/xraph/grove v1.10.2 - github.com/xraph/grove/drivers/sqlitedriver v1.10.2 + github.com/xraph/grove v1.10.3 + github.com/xraph/grove/drivers/sqlitedriver v1.10.3 ) require ( diff --git a/api/go.sum b/api/go.sum index 547979e7..6e06fe4e 100644 --- a/api/go.sum +++ b/api/go.sum @@ -291,10 +291,10 @@ github.com/xraph/forge v1.13.6 h1:JCLMd1eEd7Ay96iVg8tFlYDBMyVO8ltZSIMGRL7ktko= github.com/xraph/forge v1.13.6/go.mod h1:gg/KN2RcPOZl0DMxLRp7QQYBW3wLscXgYdUzfcpWDeI= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/xraph/shield v1.10.2 h1:LJlnTeXvDov+GBPhOGsnyoiH/1Zsx5gaOPfNfSAGoxU= github.com/xraph/shield v1.10.2/go.mod h1:xkWUxiFghxQV74UVNfDsSz/mCjQ18mpt4W4kxGiBqJU= github.com/xraph/vessel v1.0.5 h1:4In3UmIJzOrym5+FXE/uvm1jR+gofUqVmEjKjjgE3gw= diff --git a/checkpoint/checkpoint.go b/checkpoint/checkpoint.go index 46ee1184..0f16a71c 100644 --- a/checkpoint/checkpoint.go +++ b/checkpoint/checkpoint.go @@ -2,12 +2,30 @@ package checkpoint import ( + "encoding/json" "time" "github.com/xraph/cortex" "github.com/xraph/cortex/id" ) +// MetadataKind is the metadata key that names what a checkpoint the engine +// opened is waiting on. It is set to KindDeferredModelCall on a checkpoint +// opened for a deferred model call, and absent on a tool approval. +const MetadataKind = "kind" + +// KindDeferredModelCall is the MetadataKind of a checkpoint opened because +// the model client deferred a call (cortex.DeferredModelCallError). Such a +// checkpoint has no pending tool calls. An approval continues the run with +// the same model call; a rejection stops the run with the decision's +// reason. A decision that carries Calls or AmendedArguments is refused +// with cortex.ErrInvalidDecision. +const KindDeferredModelCall = "deferred_model_call" + +// MetadataDeferralKind is the metadata key that carries +// cortex.DeferredModelCallError.Kind on a deferred model call's checkpoint. +const MetadataDeferralKind = "deferral_kind" + // Decision represents the resolution of a checkpoint. type Decision struct { Approved bool `json:"approved"` @@ -23,6 +41,36 @@ type Decision struct { // Anything that serves a checkpoint to a reader goes through // Checkpoint.Public, which drops it. DecidedByPrincipal []byte `json:"decided_by_principal,omitempty"` + // AmendedArguments, on an approval, are the arguments the suspended + // call runs with instead of the model's. They replace the whole + // argument object, nothing is merged. They must be a JSON object, and + // the suspension must hold exactly one pending call, or the decision + // is refused with cortex.ErrInvalidDecision. The engine does not + // validate them against the tool's schema; that is the caller's job. + // A rejection ignores them and does not record them. + AmendedArguments json.RawMessage `json:"amended_arguments,omitempty"` + // Calls decides each pending tool call on its own, keyed by the + // pending call's id. When it is set, Approved is derived (true when + // any call is approved), AmendedArguments must be empty, and every + // pending call must have an entry and every entry must name a + // pending call, or the decision is refused with + // cortex.ErrInvalidDecision. A rejected call's Reason is what the + // model reads for that call. When it is empty, Approved and + // AmendedArguments decide every pending call together. + Calls map[string]CallDecision `json:"calls,omitempty"` +} + +// CallDecision is the outcome of one pending tool call within a Decision. +type CallDecision struct { + // Approved runs the call. A call that is not approved is rejected + // and answered with Reason. + Approved bool `json:"approved"` + // AmendedArguments, on an approved call, replace the whole argument + // object the call runs with. They must be a JSON object. A rejected + // call ignores them. + AmendedArguments json.RawMessage `json:"amended_arguments,omitempty"` + // Reason is what the model reads when the call is rejected. + Reason string `json:"reason,omitempty"` } // Checkpoint represents a point where a run pauses for approval. diff --git a/engine/continue.go b/engine/continue.go index 8964c2bb..bfcca620 100644 --- a/engine/continue.go +++ b/engine/continue.go @@ -6,6 +6,9 @@ import ( "encoding/json" "errors" "fmt" + "reflect" + "slices" + "strings" "time" "go.opentelemetry.io/otel/attribute" @@ -77,8 +80,11 @@ func (e *Engine) decideCheckpoint(ctx context.Context, cpID id.CheckpointID, d c if err != nil { return nil, err } - if !d.Approved && !e.rejectionsToModel { - if rejectErr := e.failRejected(ctx, cp, susp, d); rejectErr != nil { + if decideErr := checkDecision(susp, d); decideErr != nil { + return nil, decideErr + } + if !d.Approved && (!e.rejectionsToModel || isDeferredModelCall(susp)) { + if rejectErr := e.endRejected(ctx, cp, susp, d); rejectErr != nil { if errors.Is(rejectErr, cortex.ErrNotSuspended) && e.runMovedOn(ctx, cp) { return nil, fmt.Errorf("resolve checkpoint %s: already decided: %w", cp.ID, cortex.ErrInvalidState) } @@ -91,7 +97,10 @@ func (e *Engine) decideCheckpoint(ctx context.Context, cpID id.CheckpointID, d c if !d.Approved { source = run.ReadyRejection } - in := decisionInput(susp.Pending, d) + in, err := decisionInput(susp.Pending, d) + if err != nil { + return nil, err + } if _, previewErr := e.previewResume(ctx, cp.RunID, in, resumeSourceApproval, susp, nil); previewErr != nil { return nil, previewErr } @@ -101,7 +110,7 @@ func (e *Engine) decideCheckpoint(ctx context.Context, cpID id.CheckpointID, d c // that finds the run already ready on this decision did not. marked := err == nil if errors.Is(err, cortex.ErrNotSuspended) { - ready, err = e.retriedDecision(ctx, cp, source, err) + ready, err = e.retriedDecision(ctx, cp, source, in, d, err) } if err != nil { return nil, err @@ -205,6 +214,21 @@ func (e *Engine) openDecision(ctx context.Context, cpID id.CheckpointID, d check return nil, d, fmt.Errorf("resolve checkpoint %s: %w", cpID, err) } d.DecidedByPrincipal = decidedBy + if err := checkArgumentsObject(d.AmendedArguments); err != nil { + return nil, d, fmt.Errorf("resolve checkpoint %s: %w", cpID, err) + } + if len(d.Calls) > 0 { + d = settleCalls(d) + for callID, cd := range d.Calls { + if err := checkArgumentsObject(cd.AmendedArguments); err != nil { + return nil, d, fmt.Errorf("resolve checkpoint %s: call %q: %w", cpID, callID, err) + } + } + return cp, d, nil + } + if !d.Approved { + d.AmendedArguments = nil + } return cp, d, nil } @@ -228,7 +252,7 @@ func (e *Engine) recordDecision(ctx context.Context, cp *checkpoint.Checkpoint, // Anything else is the refusal MarkReady gave. A checkpoint decided in the // meantime is cortex.ErrInvalidState, as it would have been had this call // read it a moment later. -func (e *Engine) retriedDecision(ctx context.Context, cp *checkpoint.Checkpoint, source run.ReadySource, refused error) (*run.Run, error) { +func (e *Engine) retriedDecision(ctx context.Context, cp *checkpoint.Checkpoint, source run.ReadySource, in ResumeInput, d checkpoint.Decision, refused error) (*run.Run, error) { current, err := e.store.GetCheckpoint(ctx, cp.ID) if err != nil { return nil, fmt.Errorf("reload checkpoint %s: %w", cp.ID, err) @@ -256,6 +280,12 @@ func (e *Engine) retriedDecision(ctx context.Context, cp *checkpoint.Checkpoint, // checkpoint after the record. return nil, fmt.Errorf("resolve checkpoint %s: already decided %s: %w", cp.ID, r.Ready.Source, cortex.ErrInvalidState) } + if !sameReadyDecision(r.Ready, in, d) { + // The run is ready on this kind of decision, but not on this one: + // somebody else's edits or per-call outcomes are what will run, and + // they stand. A retry is the same decision, nothing less. + return nil, fmt.Errorf("resolve checkpoint %s: already decided differently: %w", cp.ID, cortex.ErrInvalidState) + } return r, nil } @@ -284,24 +314,165 @@ func (e *Engine) recordedAs(ctx context.Context, cp *checkpoint.Checkpoint, d ch if err != nil || current.Decision == nil { return false } - got := current.Decision - return got.Approved == d.Approved && got.DecidedBy == d.DecidedBy && bytes.Equal(got.DecidedByPrincipal, d.DecidedByPrincipal) + return sameDecision(*current.Decision, d) +} + +// sameDecision reports whether two decisions say the same thing by the +// same decider: the outcome, who decided, the edited arguments (as JSON, +// so spacing does not matter) and each call's outcome, edits and reason. +// A top-level decision and a Calls decision with one entry are the same +// shape, since a lost record is rebuilt from the run's results and cannot +// tell which of the two the decider used. +func sameDecision(a, b checkpoint.Decision) bool { + if a.Approved != b.Approved || a.DecidedBy != b.DecidedBy || !bytes.Equal(a.DecidedByPrincipal, b.DecidedByPrincipal) { + return false + } + a, b = flattenSingleCall(a), flattenSingleCall(b) + if !sameJSON(a.AmendedArguments, b.AmendedArguments) || len(a.Calls) != len(b.Calls) { + return false + } + for callID, ca := range a.Calls { + cb, ok := b.Calls[callID] + if !ok || ca.Approved != cb.Approved || ca.Reason != cb.Reason || !sameJSON(ca.AmendedArguments, cb.AmendedArguments) { + return false + } + } + return true +} + +// flattenSingleCall turns a Calls decision of one entry into the top-level +// form, whose outcome it already is. +func flattenSingleCall(d checkpoint.Decision) checkpoint.Decision { + if len(d.Calls) != 1 { + return d + } + for _, cd := range d.Calls { + d.Approved, d.AmendedArguments, d.Calls = cd.Approved, cd.AmendedArguments, nil + } + return d +} + +// sameReadyDecision reports whether the run was made ready on exactly the +// results in carries, by the same decider. A readiness that cannot be +// decoded is not the same. +func sameReadyDecision(ready *run.Readiness, in ResumeInput, d checkpoint.Decision) bool { + if ready.DecidedBy != d.DecidedBy || !bytes.Equal(ready.DecidedByPrincipal, d.DecidedByPrincipal) { + return false + } + var have []ToolResult + if err := json.Unmarshal(ready.Results, &have); err != nil || len(have) != len(in.ToolResults) { + return false + } + byID := make(map[string]ToolResult, len(have)) + for _, res := range have { + byID[res.ToolCallID] = res + } + for _, want := range in.ToolResults { + got, ok := byID[want.ToolCallID] + if !ok || got.Execute != want.Execute || got.Error != want.Error || got.Rejected != want.Rejected || got.AmendedBy != want.AmendedBy || !sameJSON(got.Arguments, want.Arguments) { + return false + } + } + return true +} + +// sameJSON reports whether two JSON documents decode to the same value. +// Two empty ones are the same. +func sameJSON(a, b json.RawMessage) bool { + if len(a) == 0 || len(b) == 0 { + return len(a) == len(b) + } + var va, vb any + if json.Unmarshal(a, &va) != nil || json.Unmarshal(b, &vb) != nil { + return bytes.Equal(a, b) + } + return reflect.DeepEqual(va, vb) } // decisionInput is what a decision hands back for each pending call. An // approval asks the engine to execute each one, since the calls were // stopped before dispatch. A rejection answers each one with an error the // model reads. -func decisionInput(pending []suspension.PendingCall, d checkpoint.Decision) ResumeInput { - in := ResumeInput{ToolResults: make([]ToolResult, 0, len(pending)), rejected: !d.Approved} +// +// Amended arguments belong to one call, so an approval that carries them +// is refused with cortex.ErrInvalidDecision unless exactly one call is +// pending. A decision with Calls decides each call on its own instead; its +// callers have already run checkCalls on it, and it needs no other +// preparation, since the outcome of each call is read from that call. +func decisionInput(pending []suspension.PendingCall, d checkpoint.Decision) (ResumeInput, error) { + in := ResumeInput{ToolResults: make([]ToolResult, 0, len(pending))} + if len(d.Calls) > 0 { + rejected := true + for _, p := range pending { + cd := d.Calls[p.ID] + if cd.Approved { + rejected = false + in.ToolResults = append(in.ToolResults, ToolResult{ToolCallID: p.ID, Execute: true, Arguments: cd.AmendedArguments, AmendedBy: amendedBy(d.DecidedBy, cd.AmendedArguments)}) + } else { + in.ToolResults = append(in.ToolResults, ToolResult{ToolCallID: p.ID, Error: rejectionMessage(checkpoint.Decision{DecidedBy: d.DecidedBy, Reason: cd.Reason}), Rejected: true}) + } + } + in.rejected = rejected + return in, nil + } + in.rejected = !d.Approved + if d.Approved && len(d.AmendedArguments) > 0 && len(pending) != 1 { + return in, fmt.Errorf("%w: amended arguments need exactly one pending call, found %d", cortex.ErrInvalidDecision, len(pending)) + } for _, p := range pending { if d.Approved { - in.ToolResults = append(in.ToolResults, ToolResult{ToolCallID: p.ID, Execute: true}) + in.ToolResults = append(in.ToolResults, ToolResult{ToolCallID: p.ID, Execute: true, Arguments: d.AmendedArguments, AmendedBy: amendedBy(d.DecidedBy, d.AmendedArguments)}) } else { in.ToolResults = append(in.ToolResults, ToolResult{ToolCallID: p.ID, Error: rejectionMessage(d)}) } } - return in + return in, nil +} + +// checkDecision reports whether d can answer susp. A deferred model call +// has no tool calls, so a decision on it approves or rejects it as a whole: +// one with Calls or amended arguments is cortex.ErrInvalidDecision. Any +// other pause is checked by checkCalls. +func checkDecision(susp *suspension.Suspension, d checkpoint.Decision) error { + if isDeferredModelCall(susp) { + if len(d.Calls) > 0 || len(d.AmendedArguments) > 0 { + return fmt.Errorf("%w: a deferred model call has no tool calls to decide or amend, so approve or reject it as a whole", cortex.ErrInvalidDecision) + } + return nil + } + return checkCalls(susp.Pending, d) +} + +// checkCalls reports whether d.Calls names exactly the pending calls, once +// each, and nothing else. +func checkCalls(pending []suspension.PendingCall, d checkpoint.Decision) error { + if len(d.Calls) == 0 { + return nil + } + if len(d.AmendedArguments) > 0 { + return fmt.Errorf("%w: amend arguments per call in Calls, not on the decision", cortex.ErrInvalidDecision) + } + known := make(map[string]bool, len(pending)) + for _, p := range pending { + known[p.ID] = true + if _, ok := d.Calls[p.ID]; !ok { + return fmt.Errorf("%w: no decision for pending call %q", cortex.ErrInvalidDecision, p.ID) + } + } + for callID := range d.Calls { + if !known[callID] { + return fmt.Errorf("%w: call %q is not pending", cortex.ErrInvalidDecision, callID) + } + } + return nil +} + +// amendedBy names who edited a call's arguments, or "" when nobody did. +func amendedBy(decidedBy string, amended json.RawMessage) string { + if len(amended) == 0 { + return "" + } + return decidedBy } // rejectionMessage is the error a rejected call returns to the model. @@ -335,6 +506,9 @@ func (e *Engine) ProvideResults(ctx context.Context, runID id.AgentRunID, in Res if _, err := cortex.RequireScope(ctx); err != nil { return err } + if err := refuseDecisionFields(in.ToolResults); err != nil { + return err + } in = ResumeInput{ToolResults: in.ToolResults} susp, err := e.store.GetSuspension(ctx, runID) if err != nil { @@ -691,6 +865,7 @@ func (e *Engine) recordLostDecision(ctx context.Context, ready *run.Readiness) { Reason: "recorded when the run continued", DecidedAt: ready.At, } + d.AmendedArguments, d.Calls = decisionFromResults(ready) if d.DecidedBy == "" && len(d.DecidedByPrincipal) == 0 { d.DecidedBy = checkpointDecidedByEngine d.Reason = "the run was continued on this decision, but who made it was not recorded" @@ -732,3 +907,72 @@ func readyInput(ready *run.Readiness) (ResumeInput, resumeSource, error) { return in, resumeSourcePublic, fmt.Errorf("unknown ready source %q: %w", ready.Source, cortex.ErrInvalidContinuation) } } + +// settleCalls derives what a per-call decision means as a whole. It is +// approved when any call is, and a rejected call keeps no edit. When every +// call is rejected and the caller gave no reason, the reasons stand in for +// it, since a run that fails on the rejection reads the decision's Reason. +// d.Calls is copied, never changed in place. +func settleCalls(d checkpoint.Decision) checkpoint.Decision { + calls := make(map[string]checkpoint.CallDecision, len(d.Calls)) + ids := make([]string, 0, len(d.Calls)) + d.Approved = false + for callID, cd := range d.Calls { + if cd.Approved { + d.Approved = true + } else { + cd.AmendedArguments = nil + } + calls[callID] = cd + ids = append(ids, callID) + } + d.Calls = calls + if !d.Approved && d.Reason == "" { + slices.Sort(ids) + reasons := make([]string, 0, len(ids)) + for _, callID := range ids { + if r := calls[callID].Reason; r != "" { + reasons = append(reasons, r) + } + } + d.Reason = strings.Join(reasons, "; ") + } + return d +} + +// decisionFromResults rebuilds what a decision edited and decided per call +// from the results its run was made ready with, for a record that was lost. +// A single call is recorded the top-level way, several are recorded in +// Calls, where a rejection keeps the reason the model was given. +func decisionFromResults(ready *run.Readiness) (amended json.RawMessage, calls map[string]checkpoint.CallDecision) { + var results []ToolResult + if json.Unmarshal(ready.Results, &results) != nil { + return nil, nil + } + if len(results) == 1 && !results[0].Rejected { + return results[0].Arguments, nil + } + perCall := false + for _, res := range results { + if res.Rejected || (res.Execute && len(res.Arguments) > 0) { + perCall = true + } + } + if !perCall { + return nil, nil + } + by := ready.DecidedBy + if by == "" { + by = "a reviewer" + } + prefix := "rejected by " + by + ": " + calls = make(map[string]checkpoint.CallDecision, len(results)) + for _, res := range results { + if res.Execute { + calls[res.ToolCallID] = checkpoint.CallDecision{Approved: true, AmendedArguments: res.Arguments} + continue + } + calls[res.ToolCallID] = checkpoint.CallDecision{Reason: strings.TrimPrefix(res.Error, prefix)} + } + return nil, calls +} diff --git a/engine/continue_amend_test.go b/engine/continue_amend_test.go new file mode 100644 index 00000000..7e9b4983 --- /dev/null +++ b/engine/continue_amend_test.go @@ -0,0 +1,447 @@ +package engine + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/xraph/cortex" + "github.com/xraph/cortex/checkpoint" + "github.com/xraph/cortex/llm" + "github.com/xraph/cortex/run" + "github.com/xraph/cortex/suspension" +) + +func (f *continueFixture) seenArguments() []string { + f.mu.Lock() + defer f.mu.Unlock() + return slices.Clone(f.args) +} + +func newAmendFixture(t *testing.T, opts ...Option) *continueFixture { + t.Helper() + ctx := cortexScopeContext() + return buildContinueFixture(ctx, t, newApprovalStore(ctx, t), []llm.ToolCall{{ID: "call-1", Name: gatedTool().Name, Arguments: `{"to":"a"}`}}, opts...) +} + +func cortexScopeContext() context.Context { + return cortex.WithScope(context.Background(), approvalScope()) +} + +func TestContinueRun_amendedArgumentsReplaceTheModelsArguments(t *testing.T) { + f := newAmendFixture(t) + paused, cp := f.pause(t) + + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{ + Approved: true, + DecidedBy: "bob", + AmendedArguments: json.RawMessage(`{"to":"b"}`), + })) + done, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, done.State) + + require.Equal(t, []string{`{"to":"b"}`}, f.seenArguments(), "the tool must run once, with the amended arguments") + + // The model sees one result for the call, and its own history shows + // the arguments that ran. + messages := f.model.lastRequest() + results := 0 + for _, m := range messages { + if m.Role == "tool" && m.ToolCallID == "call-1" { + results++ + } + for _, tc := range m.ToolCalls { + if tc.ID == "call-1" { + require.JSONEq(t, `{"to":"b"}`, tc.Arguments) + } + } + } + require.Equal(t, 1, results) + + // The decision keeps the edit, and the trace says who made it. + require.JSONEq(t, `{"to":"b"}`, string(f.checkpointState(t, cp.ID).Decision.AmendedArguments)) + steps, err := f.e.ListSteps(f.ctx, paused.ID) + require.NoError(t, err) + found := false + for _, step := range steps { + calls, listErr := f.e.ListToolCalls(f.ctx, step.ID) + require.NoError(t, listErr) + for _, call := range calls { + if call.ToolName == gatedTool().Name { + found = true + require.JSONEq(t, `{"to":"b"}`, call.Arguments) + require.Equal(t, true, call.Metadata["arguments_amended"]) + require.Equal(t, "bob", call.Metadata["amended_by"]) + } + } + } + require.True(t, found, "the approved call left no tool call row") +} + +func TestDecideCheckpoint_amendedArgumentsMustBeAnObject(t *testing.T) { + for _, bad := range []string{`[1]`, `"x"`, `null`, `{`} { + t.Run(bad, func(t *testing.T) { + f := newAmendFixture(t) + paused, cp := f.pause(t) + + err := f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{ + Approved: true, + AmendedArguments: json.RawMessage(bad), + }) + require.ErrorIs(t, err, cortex.ErrInvalidDecision) + + // The refusal leaves the checkpoint open and the run paused. + require.Equal(t, "pending", f.checkpointState(t, cp.ID).State) + require.Equal(t, run.StatePaused, f.runState(t, paused.ID).State) + }) + } +} + +func TestDecideCheckpoint_amendedArgumentsNeedOnePendingCall(t *testing.T) { + ctx := cortexScopeContext() + f := buildContinueFixture(ctx, t, newApprovalStore(ctx, t), []llm.ToolCall{ + {ID: "call-1", Name: gatedTool().Name, Arguments: `{"to":"a"}`}, + {ID: "call-2", Name: gatedTool().Name, Arguments: `{"to":"c"}`}, + }) + paused, cp := f.pause(t) + + err := f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Approved: true, AmendedArguments: json.RawMessage(`{"to":"b"}`)}) + require.ErrorIs(t, err, cortex.ErrInvalidDecision) + require.Equal(t, run.StatePaused, f.runState(t, paused.ID).State) +} + +func TestContinueRun_rejectedIgnoresAmendedArguments(t *testing.T) { + f := newAmendFixture(t, WithRejectionsReturnToModel()) + paused, cp := f.pause(t) + + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{ + DecidedBy: "bob", + Reason: "no", + AmendedArguments: json.RawMessage(`{"to":"b"}`), + })) + _, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + + require.Zero(t, f.ran.Load(), "a rejected call must never run") + content, ok := toolMessage(f.model.lastRequest()) + require.True(t, ok) + require.Contains(t, content, "rejected by bob: no") + require.Empty(t, f.checkpointState(t, cp.ID).Decision.AmendedArguments, "a rejection must not record an edit") +} + +func newTwoCallFixture(t *testing.T) *continueFixture { + t.Helper() + ctx := cortexScopeContext() + return buildContinueFixture(ctx, t, newApprovalStore(ctx, t), []llm.ToolCall{ + {ID: "call-1", Name: gatedTool().Name, Arguments: `{"to":"a"}`}, + {ID: "call-2", Name: gatedTool().Name, Arguments: `{"to":"c"}`}, + }) +} + +func TestContinueRun_perCallDecisionsApproveAmendAndReject(t *testing.T) { + f := newTwoCallFixture(t) + paused, cp := f.pause(t) + + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{ + DecidedBy: "bob", + Calls: map[string]checkpoint.CallDecision{ + "call-1": {Approved: true, AmendedArguments: json.RawMessage(`{"to":"b"}`)}, + "call-2": {Reason: "wrong recipient"}, + }, + })) + done, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, done.State) + + require.Equal(t, []string{`{"to":"b"}`}, f.seenArguments(), "only the approved call runs, with the amended arguments") + var first, second string + for _, m := range f.model.lastRequest() { + if m.Role != "tool" { + continue + } + switch m.ToolCallID { + case "call-1": + first = m.Content + case "call-2": + second = m.Content + } + } + require.Equal(t, "deleted", first) + require.Contains(t, second, "rejected by bob: wrong recipient") + require.Equal(t, 1, countEvent(f.tools.snapshot(), "completed")) + require.Equal(t, 1, countEvent(f.tools.snapshot(), "denied")) + require.Zero(t, countEvent(f.tools.snapshot(), "failed")) + + decided := f.checkpointState(t, cp.ID).Decision + require.True(t, decided.Approved, "a decision with any approved call is approved") + require.Len(t, decided.Calls, 2) + require.JSONEq(t, `{"to":"b"}`, string(decided.Calls["call-1"].AmendedArguments)) + require.Equal(t, "wrong recipient", decided.Calls["call-2"].Reason) +} + +func TestDecideCheckpoint_perCallDecisionsMustNameEveryPendingCall(t *testing.T) { + cases := map[string]map[string]checkpoint.CallDecision{ + "missing call": {"call-1": {Approved: true}}, + "unknown call": {"call-1": {Approved: true}, "call-2": {Approved: true}, "call-9": {Approved: true}}, + "wrong call id": {"call-1": {Approved: true}, "call-9": {Approved: true}}, + "amend not obj": {"call-1": {Approved: true, AmendedArguments: json.RawMessage(`[1]`)}, "call-2": {Approved: true}}, + } + for name, calls := range cases { + t.Run(name, func(t *testing.T) { + f := newTwoCallFixture(t) + paused, cp := f.pause(t) + + err := f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Calls: calls}) + require.ErrorIs(t, err, cortex.ErrInvalidDecision) + require.Equal(t, "pending", f.checkpointState(t, cp.ID).State) + require.Equal(t, run.StatePaused, f.runState(t, paused.ID).State) + }) + } +} + +func TestDecideCheckpoint_perCallDecisionsRefuseTopLevelAmendment(t *testing.T) { + f := newTwoCallFixture(t) + _, cp := f.pause(t) + err := f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{ + AmendedArguments: json.RawMessage(`{"to":"b"}`), + Calls: map[string]checkpoint.CallDecision{"call-1": {Approved: true}, "call-2": {Approved: true}}, + }) + require.ErrorIs(t, err, cortex.ErrInvalidDecision) +} + +func TestDecideCheckpoint_perCallDecisionsAllRejectedFailsRunWithoutOption(t *testing.T) { + f := newTwoCallFixture(t) + paused, cp := f.pause(t) + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{ + DecidedBy: "bob", + Calls: map[string]checkpoint.CallDecision{"call-1": {Reason: "no"}, "call-2": {Reason: "no"}}, + })) + require.Equal(t, run.StateFailed, f.runState(t, paused.ID).State) + require.Zero(t, f.ran.Load()) + require.False(t, f.checkpointState(t, cp.ID).Decision.Approved) +} + +func TestResolveCheckpoint_perCallDecisionsContinueTheRun(t *testing.T) { + f := newTwoCallFixture(t) + paused, cp := f.pause(t) + require.NoError(t, f.e.ResolveCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{ + DecidedBy: "bob", + Calls: map[string]checkpoint.CallDecision{ + "call-1": {Reason: "no"}, + "call-2": {Approved: true, AmendedArguments: json.RawMessage(`{"to":"z"}`)}, + }, + })) + require.Equal(t, run.StateCompleted, f.runState(t, paused.ID).State) + require.Equal(t, []string{`{"to":"z"}`}, f.seenArguments()) +} + +func newFlakyTwoCallFixture(t *testing.T, opts ...Option) (*continueFixture, *flakyResolveStore) { + t.Helper() + ctx := cortexScopeContext() + st := &flakyResolveStore{Store: newApprovalStore(ctx, t)} + return buildContinueFixture(ctx, t, st, []llm.ToolCall{ + {ID: "call-1", Name: gatedTool().Name, Arguments: `{"to":"a"}`}, + {ID: "call-2", Name: gatedTool().Name, Arguments: `{"to":"c"}`}, + }, opts...), st +} + +func TestDecideCheckpoint_aRetryWithDifferentEditsIsRefused(t *testing.T) { + ctx := cortexScopeContext() + st := &flakyResolveStore{Store: newApprovalStore(ctx, t)} + f := buildContinueFixture(ctx, t, st, []llm.ToolCall{{ID: "call-1", Name: gatedTool().Name, Arguments: `{"to":"a"}`}}) + paused, cp := f.pause(t) + first := checkpoint.Decision{Approved: true, DecidedBy: "bob", AmendedArguments: json.RawMessage(`{"to":"x"}`)} + + // The first decision marks the run ready and loses its record. + st.fail.Store(true) + require.Error(t, f.e.DecideCheckpoint(as(ctx, bob), cp.ID, first)) + require.Equal(t, run.StateReady, f.runState(t, paused.ID).State) + + // Different edits, or none, are not a retry of it. + for _, other := range []checkpoint.Decision{ + {Approved: true, DecidedBy: "bob", AmendedArguments: json.RawMessage(`{"to":"y"}`)}, + {Approved: true, DecidedBy: "bob"}, + } { + err := f.e.DecideCheckpoint(as(ctx, bob), cp.ID, other) + require.ErrorIs(t, err, cortex.ErrInvalidState) + require.Equal(t, checkpointStatePending, f.checkpointState(t, cp.ID).State) + } + + // The same edits with different spacing are the same decision. + same := first + same.AmendedArguments = json.RawMessage(`{ "to": "x" }`) + require.NoError(t, f.e.DecideCheckpoint(as(ctx, bob), cp.ID, same)) + _, err := f.e.ContinueRun(ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, []string{`{"to":"x"}`}, f.seenArguments()) + require.JSONEq(t, `{"to":"x"}`, string(f.checkpointState(t, cp.ID).Decision.AmendedArguments)) +} + +func TestDecideCheckpoint_aRetryWithDifferentCallOutcomesIsRefused(t *testing.T) { + f, st := newFlakyTwoCallFixture(t) + paused, cp := f.pause(t) + first := checkpoint.Decision{DecidedBy: "bob", Calls: map[string]checkpoint.CallDecision{ + "call-1": {Approved: true}, + "call-2": {Reason: "no"}, + }} + st.fail.Store(true) + require.Error(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, first)) + + for name, calls := range map[string]map[string]checkpoint.CallDecision{ + "swapped": {"call-1": {Reason: "no"}, "call-2": {Approved: true}}, + "other reason": {"call-1": {Approved: true}, "call-2": {Reason: "later"}}, + "edit added": {"call-1": {Approved: true, AmendedArguments: json.RawMessage(`{"to":"b"}`)}, "call-2": {Reason: "no"}}, + "both approved": {"call-1": {Approved: true}, "call-2": {Approved: true}}, + } { + t.Run(name, func(t *testing.T) { + err := f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{DecidedBy: "bob", Calls: calls}) + require.ErrorIs(t, err, cortex.ErrInvalidState) + }) + } + + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, first)) + _, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, []string{`{"to":"a"}`}, f.seenArguments()) +} + +func TestDecideCheckpoint_concurrentDecisionsWithDifferentEditsStoreWhatRan(t *testing.T) { + for i := range 4 { + t.Run(fmt.Sprintf("round %d", i), func(t *testing.T) { + f := newAmendFixture(t) + paused, cp := f.pause(t) + + errs := make([]error, 2) + var wg sync.WaitGroup + for j, args := range []string{`{"to":"x"}`, `{"to":"y"}`} { + wg.Go(func() { + errs[j] = f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{ + Approved: true, DecidedBy: fmt.Sprintf("op-%d", j), AmendedArguments: json.RawMessage(args), + }) + }) + } + wg.Wait() + wins := 0 + for _, err := range errs { + if err == nil { + wins++ + continue + } + require.ErrorIs(t, err, cortex.ErrInvalidState) + } + require.Equal(t, 1, wins, "exactly one decision wins: %v", errs) + + _, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + ran := f.seenArguments() + require.Len(t, ran, 1) + require.JSONEq(t, ran[0], string(f.checkpointState(t, cp.ID).Decision.AmendedArguments), "the record must match what ran") + }) + } +} + +func TestContinueRun_aLostDecisionRecordKeepsEditsAndCallOutcomes(t *testing.T) { + t.Run("amended single call", func(t *testing.T) { + ctx := cortexScopeContext() + st := &flakyResolveStore{Store: newApprovalStore(ctx, t)} + f := buildContinueFixture(ctx, t, st, []llm.ToolCall{{ID: "call-1", Name: gatedTool().Name, Arguments: `{"to":"a"}`}}) + paused, cp := f.pause(t) + st.fail.Store(true) + require.Error(t, f.e.DecideCheckpoint(as(ctx, bob), cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "bob", AmendedArguments: json.RawMessage(`{"to":"x"}`)})) + _, err := f.e.ContinueRun(ctx, paused.ID) + require.NoError(t, err) + + got := f.checkpointState(t, cp.ID).Decision + require.True(t, got.Approved) + require.JSONEq(t, `{"to":"x"}`, string(got.AmendedArguments)) + }) + t.Run("per call", func(t *testing.T) { + f, st := newFlakyTwoCallFixture(t) + paused, cp := f.pause(t) + st.fail.Store(true) + require.Error(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{DecidedBy: "bob", Calls: map[string]checkpoint.CallDecision{ + "call-1": {Approved: true, AmendedArguments: json.RawMessage(`{"to":"x"}`)}, + "call-2": {Reason: "wrong recipient"}, + }})) + _, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + + got := f.checkpointState(t, cp.ID).Decision + require.True(t, got.Approved) + require.Len(t, got.Calls, 2) + require.True(t, got.Calls["call-1"].Approved) + require.JSONEq(t, `{"to":"x"}`, string(got.Calls["call-1"].AmendedArguments)) + require.False(t, got.Calls["call-2"].Approved) + require.Equal(t, "wrong recipient", got.Calls["call-2"].Reason) + }) +} + +func TestDecisionInput_doesNotNeedASettledDecision(t *testing.T) { + pending := []suspension.PendingCall{{ID: "call-1"}, {ID: "call-2"}} + in, err := decisionInput(pending, checkpoint.Decision{DecidedBy: "bob", Calls: map[string]checkpoint.CallDecision{ + "call-1": {Approved: true}, + "call-2": {Reason: "no"}, + }}) + require.NoError(t, err) + require.False(t, in.rejected, "one call is approved, so the decision is not a whole-run rejection") + require.True(t, in.ToolResults[0].Execute) + require.False(t, in.ToolResults[1].Execute) + + in, err = decisionInput(pending, checkpoint.Decision{Calls: map[string]checkpoint.CallDecision{"call-1": {}, "call-2": {}}}) + require.NoError(t, err) + require.True(t, in.rejected) +} + +func TestPublicResumeCannotMarkADenialOrAmendArguments(t *testing.T) { + ctx := cortexScopeContext() + f := buildContinueFixture(ctx, t, newApprovalStore(ctx, t), []llm.ToolCall{{ID: "call-1", Name: externalTool().Name, Arguments: `{}`}}) + paused, _ := f.pause(t) + for name, res := range map[string]ToolResult{ + "rejected": {ToolCallID: "call-1", Error: "boom", Rejected: true}, + "arguments": {ToolCallID: "call-1", Content: "x", Arguments: json.RawMessage(`{"a":1}`)}, + "amendedby": {ToolCallID: "call-1", Content: "x", AmendedBy: "mallory"}, + } { + t.Run(name, func(t *testing.T) { + in := ResumeInput{ToolResults: []ToolResult{res}} + require.ErrorIs(t, f.e.ProvideResults(ctx, paused.ID, in), cortex.ErrInvalidDecision) + _, err := f.e.Resume(ctx, paused.ID, in) + require.ErrorIs(t, err, cortex.ErrInvalidDecision) + require.Equal(t, run.StatePaused, f.runState(t, paused.ID).State) + }) + } +} + +func TestContinueRun_perCallAllRejectedReturnsEveryReasonToTheModel(t *testing.T) { + ctx := cortexScopeContext() + f := buildContinueFixture(ctx, t, newApprovalStore(ctx, t), []llm.ToolCall{ + {ID: "call-1", Name: gatedTool().Name, Arguments: `{"to":"a"}`}, + {ID: "call-2", Name: gatedTool().Name, Arguments: `{"to":"c"}`}, + }, WithRejectionsReturnToModel()) + paused, cp := f.pause(t) + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{DecidedBy: "bob", Calls: map[string]checkpoint.CallDecision{ + "call-1": {Reason: "first reason"}, + "call-2": {Reason: "second reason"}, + }})) + require.Equal(t, run.StateReady, f.runState(t, paused.ID).State) + done, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, done.State) + + seen := map[string]string{} + for _, m := range f.model.lastRequest() { + if m.Role == "tool" { + seen[m.ToolCallID] = m.Content + } + } + require.Contains(t, seen["call-1"], "rejected by bob: first reason") + require.Contains(t, seen["call-2"], "rejected by bob: second reason") + require.Zero(t, f.ran.Load()) + require.Equal(t, 2, countEvent(f.tools.snapshot(), "denied")) + require.Zero(t, countEvent(f.tools.snapshot(), "failed")) +} diff --git a/engine/continue_test.go b/engine/continue_test.go index 91c196ce..0f91f483 100644 --- a/engine/continue_test.go +++ b/engine/continue_test.go @@ -136,6 +136,7 @@ type continueFixture struct { mu sync.Mutex ran atomic.Int32 principals []any + args []string } func newContinueFixture(t *testing.T, opts ...Option) *continueFixture { @@ -154,6 +155,7 @@ func buildContinueFixture(ctx context.Context, t *testing.T, st store.Store, fir f.ran.Add(1) f.mu.Lock() f.principals = append(f.principals, inv.Principal) + f.args = append(f.args, inv.Call.Arguments) f.mu.Unlock() return "deleted", nil }), diff --git a/engine/deferred_model_call_lease_test.go b/engine/deferred_model_call_lease_test.go new file mode 100644 index 00000000..4addfd26 --- /dev/null +++ b/engine/deferred_model_call_lease_test.go @@ -0,0 +1,97 @@ +package engine_test + +import ( + "context" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/xraph/cortex" + "github.com/xraph/cortex/checkpoint" + "github.com/xraph/cortex/engine" + "github.com/xraph/cortex/llm" + "github.com/xraph/cortex/run" +) + +// deferOnceLLM is a leaseLLM whose call number at (0-based, counting +// every call) is deferred. The deferred call reaches no response, so the +// call after it gets the response the deferred one would have had. +type deferOnceLLM struct { + *leaseLLM + at int32 + seen atomic.Int32 +} + +func (m *deferOnceLLM) Complete(ctx context.Context, req *llm.Request) (*llm.Response, error) { + if m.seen.Add(1)-1 == m.at { + return nil, cortex.DeferredModelCallError{Kind: "credits", Reason: "The workspace is out of AI credits."} + } + return m.leaseLLM.Complete(ctx, req) +} + +// A deferral pauses a leased run like any other pause: the lease and the +// progress go with it. Continued, the run is claimed under a lease, and +// when the worker carrying it out stalls in the retried model call, the +// sweep reclaims it and another replica finishes it from the pause, +// without running the tool again. +func TestModelCallDeferral_reclaimableUnderLeases(t *testing.T) { + lookup := llm.Tool{Name: "lookup", Description: "test-only tool that reads something"} + var lookups atomic.Int32 + handler := func(context.Context, cortex.Invocation) (string, error) { + lookups.Add(1) + return "found", nil + } + + inner := newLeaseLLM(toolCalls(llm.ToolCall{ID: "call-1", Name: lookup.Name, Arguments: `{}`})) + // The retried call is the leaseLLM's second. + inner.hold[1] = true + model := &deferOnceLLM{leaseLLM: inner, at: 1} + f := newLeaseFixture(t, inner, engine.WithRunLease(reclaimTTL), engine.WithLLM(model), engine.WithTool(lookup, handler)) + + paused, err := f.eng.RunAgent(leaseScope(), "assistant", "work", nil) + require.NoError(t, err) + require.Equal(t, run.StatePaused, paused.State) + require.Nil(t, f.onlyRun(t).Lease, "a paused run kept its lease") + _, err = f.store.GetProgress(leaseScope(), paused.ID) + require.ErrorIs(t, err, cortex.ErrProgressNotFound) + + cps, err := f.eng.ListPendingCheckpoints(leaseScope(), &checkpoint.ListFilter{RunID: paused.ID.String()}) + require.NoError(t, err) + require.Len(t, cps, 1) + require.Equal(t, checkpoint.KindDeferredModelCall, cps[0].Metadata[checkpoint.MetadataKind]) + require.NoError(t, f.eng.DecideCheckpoint(leaseScope(), cps[0].ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"})) + + done := make(chan runOutcome, 1) + f.runs.Add(1) + go func() { + defer f.runs.Done() + r, continueErr := f.eng.ContinueRun(leaseScope(), paused.ID) + done <- runOutcome{run: r, err: continueErr} + }() + inner.waitEntered(t) + claimed := f.onlyRun(t) + require.NotNil(t, claimed.Lease, "the continued run held no lease") + require.Equal(t, f.eng.WorkerID(), claimed.Lease.Owner) + + stall(t, f, paused.ID) + bModel := &reclaimLLM{} + b := reclaimer(t, f, bModel, newReclaimHooks(), engine.WithTool(lookup, handler)) + engine.SweepOnce(b, context.Background()) + continued, err := b.ContinueRun(leaseScope(), paused.ID) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, continued.State) + require.Equal(t, 1, bModel.calls(), "the reclaiming replica did not make exactly the retried call") + require.Equal(t, "found", lastToolMessage(t, bModel.firstRequest(t)).Content, "the tool result before the deferral was lost") + require.EqualValues(t, 1, lookups.Load(), "the tool ran again") + + inner.letGo() + if out := waitOutcome(t, done); out.err == nil { + require.Equal(t, continued.Output, out.run.Output) + } else { + require.ErrorIs(t, out.err, cortex.ErrRunLeaseLost) + } + final := f.onlyRun(t) + require.Equal(t, run.StateCompleted, final.State) + require.Equal(t, continued.Output, final.Output) +} diff --git a/engine/deferred_model_call_test.go b/engine/deferred_model_call_test.go new file mode 100644 index 00000000..bf1eb1b0 --- /dev/null +++ b/engine/deferred_model_call_test.go @@ -0,0 +1,440 @@ +package engine + +import ( + "context" + "errors" + "fmt" + "slices" + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/xraph/cortex" + "github.com/xraph/cortex/checkpoint" + "github.com/xraph/cortex/llm" + "github.com/xraph/cortex/run" + "github.com/xraph/cortex/store" + "github.com/xraph/cortex/suspension" +) + +// deferralLLM serves its scripted responses in order and answers the +// calls named in deferAt (1-based) with that error instead. A deferred +// call consumes no response, so the call after it gets the response the +// deferred one would have had. It keeps a copy of every request's +// messages. +type deferralLLM struct { + mu sync.Mutex + responses []*llm.Response + deferAt map[int]error + // onNext makes a streamed deferral surface from the stream's first + // Next rather than from opening the stream. + onNext bool + served int + requests [][]llm.Message +} + +func (m *deferralLLM) next(req *llm.Request) (*llm.Response, error) { + m.mu.Lock() + defer m.mu.Unlock() + m.requests = append(m.requests, slices.Clone(req.Messages)) + if err, ok := m.deferAt[len(m.requests)]; ok { + return nil, err + } + resp := &llm.Response{Content: "done", FinishReason: "stop", Usage: llm.Usage{TotalTokens: 7}} + if m.served < len(m.responses) { + resp = m.responses[m.served] + } + m.served++ + return resp, nil +} + +func (m *deferralLLM) Complete(_ context.Context, req *llm.Request) (*llm.Response, error) { + return m.next(req) +} + +func (m *deferralLLM) CompleteStream(_ context.Context, req *llm.Request) (llm.Stream, error) { + resp, err := m.next(req) + if err != nil && m.onNext { + return &refusedStream{err: err}, nil + } + if err != nil { + return nil, err + } + return &outcomeStream{response: resp}, nil +} + +func (m *deferralLLM) calls() int { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.requests) +} + +func (m *deferralLLM) request(t *testing.T, i int) []llm.Message { + t.Helper() + m.mu.Lock() + defer m.mu.Unlock() + require.Greater(t, len(m.requests), i, "the model got no request %d", i) + return m.requests[i] +} + +// refusedStream fails its first Next with err, before any chunk. +type refusedStream struct{ err error } + +func (s *refusedStream) Next(context.Context) (*llm.Chunk, error) { return nil, s.err } +func (*refusedStream) Close() error { return nil } +func (*refusedStream) Usage() *llm.Usage { return nil } + +func lookupTool() llm.Tool { + return llm.Tool{Name: "lookup", Description: "test-only tool that reads something"} +} + +func creditsDeferral() error { + return cortex.DeferredModelCallError{Kind: "credits", Reason: "The workspace is out of AI credits."} +} + +// deferralFixture is a continueFixture whose model calls the lookup tool +// once and then answers, deferring the calls named in deferAt. +type deferralFixture struct { + *continueFixture + model *deferralLLM + lookups atomic.Int32 +} + +func newDeferralFixture(t *testing.T, deferAt map[int]error, opts ...Option) *deferralFixture { + t.Helper() + ctx := cortex.WithScope(context.Background(), approvalScope()) + return deferralFixtureOn(ctx, t, newApprovalStore(ctx, t), deferAt, opts...) +} + +func deferralFixtureOn(ctx context.Context, t *testing.T, st store.Store, deferAt map[int]error, opts ...Option) *deferralFixture { + t.Helper() + df := &deferralFixture{model: &deferralLLM{ + responses: []*llm.Response{{ + ToolCalls: []llm.ToolCall{{ID: "call-1", Name: lookupTool().Name, Arguments: `{}`}}, + FinishReason: "tool_calls", + Usage: llm.Usage{TotalTokens: 7}, + }}, + deferAt: deferAt, + }} + base := make([]Option, 0, 2+len(opts)) + base = append(base, + WithLLM(df.model), + WithTool(lookupTool(), func(context.Context, cortex.Invocation) (string, error) { + df.lookups.Add(1) + return "found", nil + }), + ) + df.continueFixture = buildContinueFixture(ctx, t, st, nil, append(base, opts...)...) + df.overrides = &RunOverrides{MaxSteps: 5} + return df +} + +// straightRun is the same run with nothing deferred: what a deferred run +// must add up to once it is continued. +func straightRun(t *testing.T) (*run.Run, *deferralFixture) { + t.Helper() + f := newDeferralFixture(t, nil) + done, err := f.e.RunAgent(as(f.ctx, alice), "assistant", "clean up", f.overrides) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, done.State) + return done, f +} + +func TestDeferredModelCall_isTheSentinel(t *testing.T) { + err := fmt.Errorf("meter: %w", cortex.DeferredModelCallError{Kind: "credits", Reason: "out of credits"}) + require.ErrorIs(t, err, cortex.ErrDeferModelCall) + var deferred cortex.DeferredModelCallError + require.ErrorAs(t, err, &deferred) + require.Equal(t, "credits", deferred.Kind) + require.Contains(t, err.Error(), "out of credits") +} + +func TestModelCallDeferral_suspendsBeforeTheCall(t *testing.T) { + f := newDeferralFixture(t, map[int]error{2: creditsDeferral()}) + paused, cp := f.pause(t) + require.NotNil(t, cp, "a deferral opened no checkpoint") + + require.Equal(t, 2, f.model.calls(), "the model saw a call after the deferral") + require.EqualValues(t, 1, f.lookups.Load(), "the tool did not run exactly once") + + require.Equal(t, checkpoint.KindDeferredModelCall, cp.Metadata[checkpoint.MetadataKind]) + require.Equal(t, "credits", cp.Metadata[checkpoint.MetadataDeferralKind]) + require.Equal(t, "The workspace is out of AI credits.", cp.Reason) + require.Equal(t, 1, cp.StepIndex, "the checkpoint names a step other than the deferred one") + + susp, err := f.st.GetSuspension(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, suspension.ReasonDeferredModelCall, susp.Reason) + require.Empty(t, susp.Pending) + require.Equal(t, 1, susp.Cont.StepIndex, "the run would not continue at the deferred call") + require.NotNil(t, susp.ExpiresAt, "a deferral ignored the engine's suspension TTL") + + // Only the call that ran is counted. + require.Equal(t, 7, paused.TokensUsed) + require.Equal(t, 1, paused.StepCount) + require.Equal(t, 1, paused.Usage.Steps) +} + +func TestModelCallDeferral_continueRetriesTheSameCall(t *testing.T) { + straight, sf := straightRun(t) + + f := newDeferralFixture(t, map[int]error{2: creditsDeferral()}) + paused, cp := f.pause(t) + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"})) + require.Equal(t, 2, f.model.calls(), "deciding called the model") + require.Equal(t, run.StateReady, f.runState(t, paused.ID).State) + + done, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, done.State) + require.Equal(t, "done", done.Output) + + require.Equal(t, 3, f.model.calls()) + require.Equal(t, f.model.request(t, 1), f.model.request(t, 2), "the continued call is not the deferred one") + require.Equal(t, sf.model.request(t, 1), f.model.request(t, 2), "the continued call differs from a straight run's") + require.Equal(t, sf.lookups.Load(), f.lookups.Load(), "the tool ran a different number of times than in a straight run") + require.Equal(t, straight.TokensUsed, done.TokensUsed, "the deferred call was counted") + require.Equal(t, straight.StepCount, done.StepCount) + require.Equal(t, straight.Usage.Steps, done.Usage.Steps) + + decided := f.checkpointState(t, cp.ID) + require.NotEqual(t, checkpointStatePending, decided.State) + require.NotNil(t, decided.Decision) + require.True(t, decided.Decision.Approved) +} + +func TestModelCallDeferral_firstCallOfTheRun(t *testing.T) { + f := newDeferralFixture(t, map[int]error{1: fmt.Errorf("meter: %w", creditsDeferral())}) + paused, cp := f.pause(t) + require.NotNil(t, cp) + require.Equal(t, 1, f.model.calls()) + require.Zero(t, f.lookups.Load()) + require.Zero(t, paused.TokensUsed) + require.Zero(t, paused.StepCount) + require.Zero(t, cp.StepIndex) + + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"})) + done, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, done.State) + require.Equal(t, 3, f.model.calls()) + require.Equal(t, f.model.request(t, 0), f.model.request(t, 1), "the continued call is not the deferred one") + require.EqualValues(t, 1, f.lookups.Load()) + require.Equal(t, 14, done.TokensUsed) +} + +func TestModelCallDeferral_resolveCheckpointContinuesTheRun(t *testing.T) { + f := newDeferralFixture(t, map[int]error{2: creditsDeferral()}) + paused, cp := f.pause(t) + require.NoError(t, f.e.ResolveCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"})) + done := f.runState(t, paused.ID) + require.Equal(t, run.StateCompleted, done.State) + require.Equal(t, 3, f.model.calls()) + require.EqualValues(t, 1, f.lookups.Load()) +} + +func TestModelCallDeferral_deferredAgainOnContinue(t *testing.T) { + f := newDeferralFixture(t, map[int]error{2: creditsDeferral(), 3: creditsDeferral()}) + paused, cp := f.pause(t) + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"})) + again, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, run.StatePaused, again.State) + + cps, err := f.e.ListPendingCheckpoints(f.ctx, &checkpoint.ListFilter{RunID: paused.ID.String()}) + require.NoError(t, err) + require.Len(t, cps, 1) + require.NotEqual(t, cp.ID, cps[0].ID) + require.Equal(t, checkpoint.KindDeferredModelCall, cps[0].Metadata[checkpoint.MetadataKind]) + + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cps[0].ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"})) + done, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, done.State) + require.EqualValues(t, 1, f.lookups.Load()) + require.Equal(t, 14, done.TokensUsed) +} + +func TestModelCallDeferral_rejectionStopsTheRun(t *testing.T) { + for name, opts := range map[string][]Option{ + "rejections fail runs": nil, + "rejections go back to the model": {WithRejectionsReturnToModel()}, + } { + t.Run(name, func(t *testing.T) { + f := newDeferralFixture(t, map[int]error{2: creditsDeferral()}, opts...) + paused, cp := f.pause(t) + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{DecidedBy: "bob", Reason: "Cancelled by Bob."})) + + stopped := f.runState(t, paused.ID) + require.Equal(t, run.StateCancelled, stopped.State) + require.Equal(t, run.StopCancelled, stopped.StopReason) + require.Contains(t, stopped.Error, "Cancelled by Bob.") + require.Equal(t, 2, f.model.calls()) + require.Equal(t, 1, f.ended.of(paused.ID)) + + decided := f.checkpointState(t, cp.ID) + require.NotNil(t, decided.Decision) + require.False(t, decided.Decision.Approved) + require.Equal(t, "Cancelled by Bob.", decided.Decision.Reason) + + _, err := f.st.GetSuspension(f.ctx, paused.ID) + require.ErrorIs(t, err, cortex.ErrNotSuspended) + _, err = f.e.ContinueRun(f.ctx, paused.ID) + require.ErrorIs(t, err, cortex.ErrRunAlreadyDone) + }) + } +} + +func TestModelCallDeferral_refusesAnythingButAPlainDecision(t *testing.T) { + cases := map[string]checkpoint.Decision{ + "per-call decisions": {Calls: map[string]checkpoint.CallDecision{"call-1": {Approved: true}}}, + "amended arguments": {Approved: true, AmendedArguments: []byte(`{"to":"b"}`)}, + "per-call rejection": {Calls: map[string]checkpoint.CallDecision{"call-1": {Reason: "no"}}}, + } + for name, d := range cases { + t.Run(name, func(t *testing.T) { + f := newDeferralFixture(t, map[int]error{2: creditsDeferral()}) + paused, cp := f.pause(t) + d.DecidedBy = "bob" + require.ErrorIs(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, d), cortex.ErrInvalidDecision) + require.ErrorIs(t, f.e.ResolveCheckpoint(as(f.ctx, bob), cp.ID, d), cortex.ErrInvalidDecision) + require.Equal(t, checkpointStatePending, f.checkpointState(t, cp.ID).State) + require.Equal(t, run.StatePaused, f.runState(t, paused.ID).State) + require.Equal(t, 2, f.model.calls()) + }) + } +} + +func TestModelCallDeferral_onlyADecisionContinuesIt(t *testing.T) { + f := newDeferralFixture(t, map[int]error{2: creditsDeferral()}) + paused, _ := f.pause(t) + + _, err := f.e.Resume(f.ctx, paused.ID, ResumeInput{}) + require.ErrorIs(t, err, cortex.ErrDeferModelCall) + require.ErrorIs(t, f.e.ProvideResults(f.ctx, paused.ID, ResumeInput{}), cortex.ErrDeferModelCall) + _, err = f.e.ContinueRun(f.ctx, paused.ID) + require.ErrorIs(t, err, cortex.ErrRunNotContinuable) + + require.Equal(t, run.StatePaused, f.runState(t, paused.ID).State) + require.Equal(t, 2, f.model.calls()) +} + +func TestModelCallDeferral_streamingRun(t *testing.T) { + for name, onNext := range map[string]bool{"when the stream opens": false, "on the first chunk": true} { + t.Run(name, func(t *testing.T) { + f := newDeferralFixture(t, map[int]error{2: creditsDeferral()}) + f.model.onNext = onNext + + events := make(chan StreamEvent, 64) + require.NoError(t, f.e.StreamAgent(as(f.ctx, alice), "assistant", "clean up", f.overrides, events)) + last := lastEvent(events) + require.Equal(t, EventSuspended, last.Type, "the stream ended with %v", last.Data) + require.Equal(t, string(suspension.ReasonDeferredModelCall), last.Data["reason"]) + require.Equal(t, 2, f.model.calls()) + require.EqualValues(t, 1, f.lookups.Load()) + + runs, err := f.e.ListRuns(f.ctx, &run.ListFilter{Limit: 5}) + require.NoError(t, err) + require.Len(t, runs, 1) + paused := runs[0] + require.Equal(t, run.StatePaused, paused.State) + require.Equal(t, 7, paused.TokensUsed) + cps, err := f.e.ListPendingCheckpoints(f.ctx, &checkpoint.ListFilter{RunID: paused.ID.String()}) + require.NoError(t, err) + require.Len(t, cps, 1) + require.Equal(t, checkpoint.KindDeferredModelCall, cps[0].Metadata[checkpoint.MetadataKind]) + + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cps[0].ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"})) + continued := make(chan StreamEvent, 64) + require.NoError(t, f.e.ContinueRunStream(f.ctx, paused.ID, continued)) + require.Equal(t, EventDone, lastEvent(continued).Type) + + done := f.runState(t, paused.ID) + require.Equal(t, run.StateCompleted, done.State) + require.Equal(t, 3, f.model.calls()) + require.Equal(t, f.model.request(t, 1), f.model.request(t, 2)) + require.EqualValues(t, 1, f.lookups.Load()) + require.Equal(t, 14, done.TokensUsed) + }) + } +} + +// lastEvent drains events and returns the last one. +func lastEvent(events <-chan StreamEvent) StreamEvent { + var last StreamEvent + for ev := range events { + last = ev + } + return last +} + +// A model error that is not a deferral still fails the run. +func TestModelCallDeferral_otherErrorsStillFail(t *testing.T) { + f := newDeferralFixture(t, map[int]error{2: errors.New("provider down")}) + _, err := f.e.RunAgent(as(f.ctx, alice), "assistant", "clean up", f.overrides) + require.Error(t, err) + require.NotErrorIs(t, err, cortex.ErrDeferModelCall) + runs, listErr := f.e.ListRuns(f.ctx, &run.ListFilter{Limit: 5}) + require.NoError(t, listErr) + require.Len(t, runs, 1) + require.Equal(t, run.StateFailed, runs[0].State) +} + +// A store that cannot hold ready runs resolves a deferral in place: an +// approval makes the call on the caller's goroutine, a rejection stops the +// run, and anything but a plain decision is refused. +func TestModelCallDeferral_resolvedInPlace(t *testing.T) { + inPlace := func(t *testing.T) *deferralFixture { + ctx := cortex.WithScope(context.Background(), approvalScope()) + return deferralFixtureOn(ctx, t, hiddenContinuationStore{newApprovalStore(ctx, t)}, map[int]error{2: creditsDeferral()}) + } + + t.Run("approved", func(t *testing.T) { + f := inPlace(t) + paused, cp := f.pause(t) + require.NoError(t, f.e.ResolveCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"})) + require.Equal(t, run.StateCompleted, f.runState(t, paused.ID).State) + require.Equal(t, 3, f.model.calls()) + require.Equal(t, f.model.request(t, 1), f.model.request(t, 2)) + require.EqualValues(t, 1, f.lookups.Load()) + require.True(t, f.checkpointState(t, cp.ID).Decision.Approved) + }) + t.Run("rejected", func(t *testing.T) { + f := inPlace(t) + paused, cp := f.pause(t) + require.NoError(t, f.e.ResolveCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{DecidedBy: "bob", Reason: "Cancelled by Bob."})) + stopped := f.runState(t, paused.ID) + require.Equal(t, run.StateCancelled, stopped.State) + require.Contains(t, stopped.Error, "Cancelled by Bob.") + require.Equal(t, 2, f.model.calls()) + require.False(t, f.checkpointState(t, cp.ID).Decision.Approved) + }) + t.Run("amended", func(t *testing.T) { + f := inPlace(t) + paused, cp := f.pause(t) + err := f.e.ResolveCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "bob", AmendedArguments: []byte(`{"to":"b"}`)}) + require.ErrorIs(t, err, cortex.ErrInvalidDecision) + require.Equal(t, run.StatePaused, f.runState(t, paused.ID).State) + require.Equal(t, checkpointStatePending, f.checkpointState(t, cp.ID).State) + }) +} + +// An empty allowlist and no allowlist freeze to the same tools: a run +// started with Tools set to an empty slice can be continued, although the +// continuation stores no tools at all. +func TestModelCallDeferral_continuesARunWithAnEmptyToolList(t *testing.T) { + f := newDeferralFixture(t, map[int]error{1: creditsDeferral()}) + f.model.responses = nil + f.overrides = &RunOverrides{MaxSteps: 5, Tools: []string{}, ToolsRestricted: true} + paused, cp := f.pause(t) + require.NotNil(t, cp) + + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"})) + done, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, done.State) + require.Equal(t, "done", done.Output) + require.Zero(t, f.lookups.Load()) +} diff --git a/engine/engine.go b/engine/engine.go index 422cfa7b..801b8022 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -939,6 +939,9 @@ func (e *Engine) ResolveCheckpoint(ctx context.Context, cpID id.CheckpointID, de // that cannot be read is treated as one that continues; decideCheckpoint // reports the read error itself. func (e *Engine) decisionContinues(ctx context.Context, cpID id.CheckpointID, d checkpoint.Decision) bool { + if len(d.Calls) > 0 { + d = settleCalls(d) + } if !d.Approved && !e.rejectionsToModel { return false } @@ -984,7 +987,10 @@ func (e *Engine) resolveInPlace(ctx context.Context, cpID id.CheckpointID, decis if bindErr != nil { return bindErr } - if decision.Approved || e.rejectionsToModel { + if err := checkDecision(susp, decision); err != nil { + return err + } + if decision.Approved || (e.rejectionsToModel && !isDeferredModelCall(susp)) { if err := e.resumeDecided(ctx, cp, susp, decision); err != nil { uncertain := errors.Is(err, cortex.ErrEffectUncertain) exhausted := errors.Is(err, cortex.ErrMaxStepsReached) @@ -1006,7 +1012,7 @@ func (e *Engine) resolveInPlace(ctx context.Context, cpID id.CheckpointID, decis } executionErr = err } - } else if err := e.failRejected(ctx, cp, susp, decision); err != nil { + } else if err := e.endRejected(ctx, cp, susp, decision); err != nil { return err } } @@ -1064,7 +1070,11 @@ func (e *Engine) resumeDecided(ctx context.Context, cp *checkpoint.Checkpoint, s // resume rather than Resume: this is the one caller allowed to say a // checkpoint decided these calls, and it says so having just read a // pending checkpoint for this run. - if _, err := e.resume(ctx, cp.RunID, decisionInput(susp.Pending, decision), resumeSourceApproval); err != nil { + in, err := decisionInput(susp.Pending, decision) + if err != nil { + return err + } + if _, err := e.resume(ctx, cp.RunID, in, resumeSourceApproval); err != nil { return fmt.Errorf("resume decided run %s: %w", cp.RunID, err) } return nil @@ -1111,7 +1121,7 @@ func (e *Engine) failRejected(ctx context.Context, cp *checkpoint.Checkpoint, bo return fmt.Errorf("load run for rejected checkpoint: %w", err) } - susp, err := e.claimRejected(ctx, cp, bound, snapshot) + susp, err := e.claimRejected(ctx, cp, bound, snapshot, nil) if err != nil { return fmt.Errorf("claim suspension for rejected checkpoint %s: %w", cp.ID, err) } @@ -1145,13 +1155,17 @@ func (e *Engine) failRejected(ctx context.Context, cp *checkpoint.Checkpoint, bo // claimRejected claims the pause a rejection answers. With a versioned // store the claim takes bound and nothing else; a run waiting on another -// pause is cortex.ErrInvalidState. -func (e *Engine) claimRejected(ctx context.Context, cp *checkpoint.Checkpoint, bound *suspension.Suspension, snapshot *run.Run) (*suspension.Suspension, error) { +// pause is cortex.ErrInvalidState. ending, when set, goes on the leased +// claim's snapshot, so a worker that reclaims the run after a lost write +// finishes that ending instead of continuing the run. +func (e *Engine) claimRejected(ctx context.Context, cp *checkpoint.Checkpoint, bound *suspension.Suspension, snapshot *run.Run, ending *suspension.Ending) (*suspension.Suspension, error) { // A claim that takes the lease leaves a lease that lapses if the // failure below is never written, and the run is reclaimed from the // state it paused with: the person is asked again. if bound != nil { - if susp, leased, err := e.claimPauseLeased(ctx, snapshot, bound, time.Time{}, claimSnapshot(bound, nil)); leased { + first := claimSnapshot(bound, nil) + first.Ending = ending + if susp, leased, err := e.claimPauseLeased(ctx, snapshot, bound, time.Time{}, first); leased { return e.rejectedClaimResult(ctx, cp, bound, susp, err) } } @@ -1168,6 +1182,59 @@ func (e *Engine) claimRejected(ctx context.Context, cp *checkpoint.Checkpoint, b return e.rejectedClaimResult(ctx, cp, bound, susp, err) } +// endRejected ends the run behind a rejected checkpoint when the rejection +// leaves nothing to continue: a deferred model call is stopped, any other +// pause is failed. +func (e *Engine) endRejected(ctx context.Context, cp *checkpoint.Checkpoint, bound *suspension.Suspension, decision checkpoint.Decision) error { + if isDeferredModelCall(bound) { + return e.stopDeferred(ctx, cp, bound, decision) + } + return e.failRejected(ctx, cp, bound, decision) +} + +// stopDeferred ends a run whose deferred model call was rejected, which is +// how a host stops a run that waits on one: the run is cancelled, with the +// decider's reason as its error, and the pause is dropped. The deferred +// call is never made. +// +// It claims the pause the way failRejected does, so a decision racing +// another one, or a cancel, is refused rather than written over it. A +// leased claim records the cancel on its snapshot, so a worker that +// reclaims the run after a lost write finishes the cancel rather than +// making the call. The checkpoint is left for the caller to record the +// decision on. +func (e *Engine) stopDeferred(ctx context.Context, cp *checkpoint.Checkpoint, bound *suspension.Suspension, decision checkpoint.Decision) error { + snapshot, err := e.store.GetRun(ctx, cp.RunID) + if err != nil { + return fmt.Errorf("load run for rejected checkpoint: %w", err) + } + susp, err := e.claimRejected(ctx, cp, bound, snapshot, &suspension.Ending{Kind: suspension.EndingCancel}) + if err != nil { + return fmt.Errorf("claim suspension for rejected checkpoint %s: %w", cp.ID, err) + } + ctx = cortex.WithScope(ctx, susp.Scope) + r := e.claimedRun(ctx, cp.RunID, snapshot, susp.ClaimedRun) + + reason := decision.Reason + if reason == "" { + reason = "no reason given" + } + r.State = run.StateCancelled + r.StopReason = run.StopCancelled + r.Error = "stopped while its model call was deferred: " + reason + completedAt := time.Now().UTC() + r.CompletedAt = &completedAt + if err := e.persistRun(context.WithoutCancel(ctx), r); err != nil { + // The suspension and the checkpoint stay, as in failRejected. + return fmt.Errorf("stop run %s: %w", cp.RunID, err) + } + e.emitTerminalOutcome(context.WithoutCancel(ctx), r) + if err := e.store.DeleteSuspension(ctx, cp.RunID); err != nil && !errors.Is(err, cortex.ErrNotSuspended) { + e.logError("delete suspension of a stopped run", err, log.String("run_id", cp.RunID.String())) + } + return nil +} + // rejectedClaimResult maps a pinned rejection claim's refusal: a run // waiting on another pause is cortex.ErrInvalidState. func (e *Engine) rejectedClaimResult(ctx context.Context, cp *checkpoint.Checkpoint, bound, susp *suspension.Suspension, err error) (*suspension.Suspension, error) { diff --git a/engine/lease.go b/engine/lease.go index a1267490..d5423104 100644 --- a/engine/lease.go +++ b/engine/lease.go @@ -308,7 +308,7 @@ func answeredCalls(pending []suspension.PendingCall, in ResumeInput) []suspensio continue } switch { - case res.Error != "" && in.rejected: + case res.Error != "" && (in.rejected || res.Rejected): done = append(done, suspension.ProgressResult{CallID: p.ID, Content: jsonResult("error", res.Error), Error: res.Error, Outcome: progressOutcome(outcomeDenied)}) case res.Error != "": done = append(done, suspension.ProgressResult{CallID: p.ID, Content: jsonResult("error", res.Error), Error: res.Error, Outcome: progressOutcome(outcomeFailed)}) diff --git a/engine/prepared_resume.go b/engine/prepared_resume.go index 85b43aae..156274f1 100644 --- a/engine/prepared_resume.go +++ b/engine/prepared_resume.go @@ -59,6 +59,15 @@ func validateFrozenContinuation(cont suspension.Continuation, snap *preparedSnap return cortex.ErrInvalidContinuation } cfg.Budget.Deadline = frozen.Budget.Deadline + // The continuation drops an empty tool list (omitempty) while the + // snapshot keeps it. Both mean no listed tools; ToolsRestricted says + // whether that is an explicit allowlist, and it is compared as is. + if len(cfg.Tools) == 0 { + cfg.Tools = nil + } + if len(frozen.Tools) == 0 { + frozen.Tools = nil + } if !reflect.DeepEqual(cfg, frozen) || cont.SystemPrompt != prompt.Assemble(snap.Sections) || r.AgentID != snap.Agent.ID || r.Scope.Canonical() != snap.Scope.Canonical() { return fmt.Errorf("cortex: continuation changed frozen inputs: %w", cortex.ErrInvalidContinuation) } @@ -82,7 +91,8 @@ func (e *Engine) authorizeResumedCalls(ctx context.Context, r *run.Run, susp *su byID[result.ToolCallID] = result } for _, pending := range susp.Pending { - if !byID[pending.ID].Execute { + result := byID[pending.ID] + if !result.Execute { continue } if !susp.Cont.Config.Budget.Deadline.IsZero() && !time.Now().Before(susp.Cont.Config.Budget.Deadline) { @@ -91,7 +101,7 @@ func (e *Engine) authorizeResumedCalls(ctx context.Context, r *run.Run, susp *su if st.preparedSnapshot != nil && len(filterByName(st.preparedSnapshot.Tools, []string{pending.Name})) == 0 { return fmt.Errorf("cortex: resumed tool is outside prepared allowlist: %w", cortex.ErrInvalidContinuation) } - if err := e.authorizeResumedTool(ctx, subject, llm.ToolCall{ID: pending.ID, Name: pending.Name, Arguments: pending.Arguments}, susp.Reason == suspension.ReasonApproval); err != nil { + if err := e.authorizeResumedTool(ctx, subject, llm.ToolCall{ID: pending.ID, Name: pending.Name, Arguments: resumedArguments(pending, result)}, susp.Reason == suspension.ReasonApproval); err != nil { return err } } diff --git a/engine/react.go b/engine/react.go index 1e95e7d2..cf38600b 100644 --- a/engine/react.go +++ b/engine/react.go @@ -244,6 +244,18 @@ func (e *Engine) continueReAct(ctx context.Context, ag *agent.Config, cfg resolv fresh := progressCont(r, st, cfg) return r, e.leaveRun(ctx, r, &fresh) } + if errors.Is(err, cortex.ErrDeferModelCall) { + // The client will make this call later, not now: the run + // pauses before it, and a decision continues it with the + // same call. Nothing is charged for a call that did not run. + if _, suspendErr := e.suspendDeferred(ctx, r, st, cfg, err); suspendErr != nil { + e.failRun(ctx, r, ag.ID, suspendErr, startedAt) + stepSpan.finish(stepError, suspendErr) + return nil, fmt.Errorf("suspend run: %w", suspendErr) + } + stepSpan.finish(stepSuspended, nil) + return r, nil + } if errors.Is(err, llm.ErrContentBlocked) { reported := 0 if resp != nil { @@ -606,6 +618,11 @@ func (e *Engine) continueStreamReAct(ctx context.Context, ag *agent.Config, cfg sendLeftEvent(ctx, events, r, e.leaveRun(ctx, r, &fresh)) return } + if errors.Is(err, cortex.ErrDeferModelCall) { + // Same pause as the synchronous loop. + e.suspendDeferredStream(ctx, ag, r, st, cfg, err, stepSpan, startedAt, events) + return + } if errors.Is(err, llm.ErrContentBlocked) { terminalErr := e.stopContentBlocked(ctx, r, st, cfg, 0) stepSpan.finish(stepStopped, nil) @@ -685,6 +702,16 @@ func (e *Engine) continueStreamReAct(ctx context.Context, ag *agent.Config, cfg sendLeftEvent(ctx, events, r, e.leaveRun(ctx, r, &fresh)) return } + if errors.Is(err, cortex.ErrDeferModelCall) && totalBytes == 0 { + // A client may defer the call from the stream's first + // read rather than from opening it. Nothing of the + // response was read, so the run pauses before the call as + // it does above. Once a chunk has arrived the call has + // run, and a deferral fails the run like any error. + _ = stream.Close() + e.suspendDeferredStream(ctx, ag, r, st, cfg, err, stepSpan, startedAt, events) + return + } if errors.Is(err, llm.ErrContentBlocked) { // Nothing from a blocked stream runs: the tool-call deltas // already read are dropped with it. @@ -994,6 +1021,26 @@ func (e *Engine) continueStreamReAct(ctx context.Context, ag *agent.Config, cfg sendEvent(ctx, events, outcomeEvent(r, completedAt.Sub(startedAt))) } +// suspendDeferredStream is the streaming loop's pause for a deferred +// model call: suspendDeferred, and the suspended event that tells the +// consumer the run paused rather than ended. A pause that cannot be +// written fails the run. +func (e *Engine) suspendDeferredStream(ctx context.Context, ag *agent.Config, r *run.Run, st *reactState, cfg resolvedConfig, cause error, stepSpan *stepTrace, startedAt time.Time, events chan<- StreamEvent) { + reason, err := e.suspendDeferred(ctx, r, st, cfg, cause) + if err != nil { + e.failRun(ctx, r, ag.ID, err, startedAt) + stepSpan.finish(stepError, err) + sendEvent(ctx, events, runErrorEvent(r, err)) + return + } + stepSpan.finish(stepSuspended, nil) + sendEvent(ctx, events, StreamEvent{Type: EventSuspended, Data: map[string]any{ + "run_id": r.ID.String(), + "reason": string(reason), + "pending": publicPendingCalls(r, nil), + }}) +} + // sendLeftEvent tells a stream's consumer how a pass that left its run // ended: the run's own outcome when it turned out to have ended already // (err is nil), otherwise the hand-off or the lost lease. diff --git a/engine/resume.go b/engine/resume.go index 79fc9223..19b3eaca 100644 --- a/engine/resume.go +++ b/engine/resume.go @@ -2,6 +2,7 @@ package engine import ( "context" + "encoding/json" "errors" "fmt" "reflect" @@ -58,6 +59,16 @@ type ToolResult struct { // external tool is answered by supplying its result, since the engine // is not the thing that runs it. Execute bool `json:"execute,omitempty"` + // Arguments, with Execute, replace the arguments the call was + // suspended with: a decider edited them. They must be a JSON object. + // Only an approval decision sets them. + Arguments json.RawMessage `json:"arguments,omitempty"` + // AmendedBy says who edited Arguments. It goes on the call's trace. + AmendedBy string `json:"amended_by,omitempty"` + // Rejected says this one call is a reviewer's rejection going back to + // the model, in a decision that approved other calls. It is recorded + // as denied rather than failed, as a whole-run rejection is. + Rejected bool `json:"rejected,omitempty"` } // ResumeInput is everything a caller supplies to continue a suspended @@ -99,6 +110,9 @@ type resumption struct { // or a rebuilt message list are three different ways to corrupt a run // that looked fine at the call site and went wrong several turns later. func (e *Engine) Resume(ctx context.Context, runID id.AgentRunID, in ResumeInput) (*run.Run, error) { + if err := refuseDecisionFields(in.ToolResults); err != nil { + return nil, err + } return e.resume(ctx, runID, in, resumeSourcePublic) } @@ -175,6 +189,10 @@ func (e *Engine) resumeRun(ctx context.Context, runID id.AgentRunID, in ResumeIn // when execution completes, same as StreamAgent's, and the run's admission // slot is held until then. func (e *Engine) ResumeStream(ctx context.Context, runID id.AgentRunID, in ResumeInput, events chan<- StreamEvent) (err error) { + if err = refuseDecisionFields(in.ToolResults); err != nil { + close(events) + return err + } ctx, done, err := e.admit(ctx) if err != nil { close(events) @@ -722,6 +740,11 @@ func (e *Engine) failResume(ctx context.Context, r *run.Run, agentID id.AgentID, // exception to completed or failed: the call was refused by a person, so // it emits ToolDenied, the event an authorizer's refusal emits. func (e *Engine) recordResumedCalls(ctx context.Context, ag *agent.Config, r *run.Run, susp *suspension.Suspension, in ResumeInput, st *reactState, cfg resolvedConfig) error { + if len(susp.Pending) == 0 { + // A deferred model call: nothing to record, the loop makes the + // call next. + return nil + } ctx = withOperationalRuntime(ctx, r, cfg, st) byID := make(map[string]ToolResult, len(in.ToolResults)) for _, res := range in.ToolResults { @@ -774,6 +797,10 @@ func (e *Engine) recordResumedCalls(ctx context.Context, ag *agent.Config, r *ru } } completedAt := time.Now().UTC() + if res.Execute && len(res.Arguments) > 0 { + // The model's history says what ran, not what it first asked for. + amendMessageArguments(st.messages, p.ID, string(res.Arguments)) + } if !stepID.IsNil() { toolCall := &run.ToolCall{ @@ -782,7 +809,7 @@ func (e *Engine) recordResumedCalls(ctx context.Context, ag *agent.Config, r *ru StepID: stepID, RunID: r.ID, ToolName: publicToolName(ctx, p.Name), - Arguments: publicGenerationText(r, p.Arguments), + Arguments: publicGenerationText(r, resumedArguments(p, res)), Result: publicGenerationText(r, content), Error: publicToolErrorText(ctx, errText), // The call really was outstanding from the moment the run @@ -791,6 +818,9 @@ func (e *Engine) recordResumedCalls(ctx context.Context, ag *agent.Config, r *ru StartedAt: &startedAt, CompletedAt: &completedAt, } + if res.Execute && len(res.Arguments) > 0 { + toolCall.Metadata = map[string]any{"arguments_amended": true, "amended_by": res.AmendedBy} + } if err := e.store.CreateToolCall(ctx, toolCall); err != nil { e.logError("create resumed tool call", err, log.String("run_id", r.ID.String())) } @@ -801,7 +831,7 @@ func (e *Engine) recordResumedCalls(ctx context.Context, ag *agent.Config, r *ru source = verification.ToolLocal } switch { - case in.rejected && errText != "": + case (in.rejected || res.Rejected) && errText != "": outcome, source = outcomeDenied, verification.ToolLocal e.emitToolDenied(ctx, r.ID, p.Name, errText) case errText != "": @@ -820,6 +850,42 @@ func (e *Engine) recordResumedCalls(ctx context.Context, ag *agent.Config, r *ru return nil } +// resumedArguments is what an approved call runs with: the arguments a +// decider amended, if they did, else the ones the model asked for. +func resumedArguments(p suspension.PendingCall, res ToolResult) string { + if res.Execute && len(res.Arguments) > 0 { + return string(res.Arguments) + } + return p.Arguments +} + +// refuseDecisionFields refuses results that carry what only a checkpoint +// decision may set: edited arguments, who edited them, and the mark of a +// reviewer's denial. A public caller must not be able to have an ordinary +// tool error recorded and emitted as a person's rejection. +func refuseDecisionFields(results []ToolResult) error { + for _, r := range results { + if r.Rejected || r.AmendedBy != "" || len(r.Arguments) > 0 { + return fmt.Errorf("%w: arguments and denials are set by a checkpoint decision, not by a caller's results", cortex.ErrInvalidDecision) + } + } + return nil +} + +// checkArgumentsObject reports whether amended arguments, when there are +// any, are a JSON object. Validating them against the tool's schema is +// left to the caller. +func checkArgumentsObject(args json.RawMessage) error { + if len(args) == 0 { + return nil + } + var obj map[string]json.RawMessage + if err := json.Unmarshal(args, &obj); err != nil || obj == nil { + return fmt.Errorf("%w: amended arguments must be a JSON object", cortex.ErrInvalidDecision) + } + return nil +} + // resumedCallResult returns checked-call content, error text and host outcome. // It is also where an approved call is finally executed. // @@ -847,7 +913,7 @@ func (e *Engine) resumedCallResult(ctx context.Context, s cortex.Subject, p susp return jsonResult("error", err.Error()), err.Error(), outcomeFailed } - call := llm.ToolCall{ID: p.ID, Name: p.Name, Arguments: p.Arguments} + call := llm.ToolCall{ID: p.ID, Name: p.Name, Arguments: resumedArguments(p, res)} content, outcome, _ = e.traceTool(ctx, call, func(ctx context.Context) (string, toolOutcome, pendingWait) { out, callOutcome := e.runResumedTool(ctx, s, call, approved, &errText) return out, callOutcome, pendingWait{} @@ -954,6 +1020,15 @@ func checkResumeAuthority(reason suspension.SuspendReason, source resumeSource) } return fmt.Errorf("%w: this run is waiting on another agent's reply, which only the messaging bus can deliver", ErrNotAgentReplyResumable) + case suspension.ReasonDeferredModelCall: + // Nothing outside the engine has a result to give a deferred + // model call. The run continues when its checkpoint is decided, + // and then it makes the call itself. + if source == resumeSourceApproval { + return nil + } + return fmt.Errorf("%w: this run is waiting to make a model call it deferred, so decide its checkpoint rather than resuming the run", cortex.ErrDeferModelCall) + default: return nil } @@ -977,6 +1052,12 @@ func validateResults(pending []suspension.PendingCall, results []ToolResult) err if !ok { return fmt.Errorf("%w: unexpected result for call id %q", cortex.ErrResultsMismatch, r.ToolCallID) } + if len(r.Arguments) > 0 && !r.Execute { + return fmt.Errorf("%w: arguments for call id %q without execute", cortex.ErrInvalidDecision, r.ToolCallID) + } + if err := checkArgumentsObject(r.Arguments); err != nil { + return err + } if n > 0 { return fmt.Errorf("%w: duplicate result for call id %q", cortex.ErrResultsMismatch, r.ToolCallID) } @@ -989,3 +1070,16 @@ func validateResults(pending []suspension.PendingCall, results []ToolResult) err } return nil } + +// amendMessageArguments rewrites the arguments of the assistant tool call +// callID in messages, so the conversation the model reads next matches +// the call that ran. +func amendMessageArguments(messages []llm.Message, callID, args string) { + for i := range messages { + for j := range messages[i].ToolCalls { + if messages[i].ToolCalls[j].ID == callID { + messages[i].ToolCalls[j].Arguments = args + } + } + } +} diff --git a/engine/suspend.go b/engine/suspend.go index c0756f85..7e9a194c 100644 --- a/engine/suspend.go +++ b/engine/suspend.go @@ -86,6 +86,11 @@ type pendingCalls struct { // run is the run this step belongs to, and the only one whose asks // it collects. run id.AgentRunID + + // deferred is set, and calls empty, when the step's model call was + // deferred by the model client rather than answered: the run waits + // before that call, with nothing pending. + deferred *cortex.DeferredModelCallError } func (p *pendingCalls) add(tc llm.ToolCall, reason suspension.SuspendReason, ttl time.Duration, note string) { @@ -134,6 +139,9 @@ func stepPendingCalls(ctx context.Context, runID id.AgentRunID) *pendingCalls { // asked the engine to run, three steps away from anything that explains // it. func (p *pendingCalls) reason() (suspension.SuspendReason, error) { + if p.deferred != nil { + return suspension.ReasonDeferredModelCall, nil + } if len(p.reasons) == 0 { return "", errors.New("suspend was asked for with no pending calls") } @@ -245,7 +253,7 @@ func (e *Engine) suspendRun(ctx context.Context, r *run.Run, pend pendingCalls, // decision nobody can see: ListPending never shows it, so nothing // resolves it. Written here, a failed checkpoint write takes the // suspension down with it and the run never reaches paused at all. - cp, err := e.createCheckpoint(ctx, r, s, reason) + cp, err := e.createCheckpoint(ctx, r, s, reason, pend.deferred) if err != nil { if delErr := e.store.DeleteSuspension(ctx, r.ID); delErr != nil { e.logError("delete suspension after a failed checkpoint write", delErr, log.String("run_id", r.ID.String())) @@ -378,6 +386,12 @@ func (e *Engine) suspensionDeadline(pend pendingCalls, reason suspension.Suspend } var earliest *time.Time + if pend.deferred != nil && e.suspensionTTL > 0 { + // A deferred model call has no pending call to bring a TTL of its + // own, so it waits as long as the engine's suspension TTL allows. + due := now.Add(e.suspensionTTL).UTC() + earliest = &due + } for i, call := range pend.calls { var due time.Time var ttl time.Duration @@ -418,7 +432,10 @@ func (e *Engine) suspensionDeadline(pend pendingCalls, reason suspension.Suspend // answered by the host that registered the tool, not by a person, and a // checkpoint queue full of rows no human is meant to decide is a queue // people stop reading. -func (e *Engine) createCheckpoint(ctx context.Context, r *run.Run, s *suspension.Suspension, reason suspension.SuspendReason) (*checkpoint.Checkpoint, error) { +func (e *Engine) createCheckpoint(ctx context.Context, r *run.Run, s *suspension.Suspension, reason suspension.SuspendReason, deferred *cortex.DeferredModelCallError) (*checkpoint.Checkpoint, error) { + if reason == suspension.ReasonDeferredModelCall && deferred != nil { + return e.createDeferralCheckpoint(ctx, r, s, *deferred) + } if reason != suspension.ReasonApproval { return nil, nil } @@ -468,3 +485,63 @@ func (e *Engine) createCheckpoint(ctx context.Context, r *run.Run, s *suspension // the pause is real. return cp, nil } + +// deferredCallReason is the checkpoint reason of a deferred model call +// whose client gave none. +const deferredCallReason = "the model call was deferred" + +// createDeferralCheckpoint opens the checkpoint of a deferred model call: +// what the host decides on to continue the run (approve, and the same +// call is made) or to stop it (reject). The checkpoint carries the +// client's reason, and its metadata names the kind of checkpoint and the +// kind of wait, so a host can tell it from a tool approval and knows what +// it waits on. +func (e *Engine) createDeferralCheckpoint(ctx context.Context, r *run.Run, s *suspension.Suspension, deferred cortex.DeferredModelCallError) (*checkpoint.Checkpoint, error) { + reason := deferred.Reason + if reason == "" { + reason = deferredCallReason + } + cp := &checkpoint.Checkpoint{ + Entity: cortex.NewEntity(), + ID: id.NewCheckpointID(), + RunID: r.ID, + AgentID: r.AgentID, + Scope: r.Scope, + Reason: reason, + // The deferred call is the next step's, the one the run continues + // with. + StepIndex: s.Cont.StepIndex, + State: checkpointStatePending, + Metadata: map[string]any{ + checkpointSuspensionKey: s.ID.String(), + checkpoint.MetadataKind: checkpoint.KindDeferredModelCall, + checkpoint.MetadataDeferralKind: deferred.Kind, + }, + } + if err := e.store.CreateCheckpoint(ctx, cp); err != nil { + return nil, err + } + return cp, nil +} + +// suspendDeferred pauses a run whose model client deferred the step's +// model call (cortex.ErrDeferModelCall). The continuation is the state +// from before that call, so nothing the run already did is repeated when +// it continues, and the deferred call itself is not counted: it has not +// run yet. A decision on the checkpoint continues the run, which makes the +// same call again. +func (e *Engine) suspendDeferred(ctx context.Context, r *run.Run, st *reactState, cfg resolvedConfig, cause error) (suspension.SuspendReason, error) { + var deferred cortex.DeferredModelCallError + if !errors.As(cause, &deferred) { + // A client that wrapped the bare sentinel said nothing more. + deferred = cortex.DeferredModelCallError{} + } + st.provenance = r.Provenance + return e.suspend(ctx, r, pendingCalls{run: r.ID, deferred: &deferred}, st.continuation(cfg)) +} + +// isDeferredModelCall reports whether a suspension waits on a deferred +// model call rather than on tool calls. +func isDeferredModelCall(susp *suspension.Suspension) bool { + return susp != nil && susp.Reason == suspension.ReasonDeferredModelCall +} diff --git a/engine/sweeper.go b/engine/sweeper.go index ea818227..0382425e 100644 --- a/engine/sweeper.go +++ b/engine/sweeper.go @@ -549,5 +549,8 @@ func expiredDetail(susp *suspension.Suspension) string { if susp.ExpiresAt != nil { deadline = susp.ExpiresAt.Format(time.RFC3339) } + if isDeferredModelCall(susp) { + return "nothing continued the deferred model call by " + deadline + } return fmt.Sprintf("nothing answered the %s pending call(s) by %s", susp.Reason, deadline) } diff --git a/engine/telemetry.go b/engine/telemetry.go index ca41f71d..23935140 100644 --- a/engine/telemetry.go +++ b/engine/telemetry.go @@ -275,7 +275,7 @@ func purposeLabel(p run.ExecutionPurpose) string { func suspendReasonLabel(r suspension.SuspendReason) string { switch r { - case suspension.ReasonApproval, suspension.ReasonExternalTool, suspension.ReasonAgentReply: + case suspension.ReasonApproval, suspension.ReasonExternalTool, suspension.ReasonAgentReply, suspension.ReasonDeferredModelCall: return string(r) default: return labelUnknown diff --git a/errors.go b/errors.go index 2f1f93a1..248ddd8f 100644 --- a/errors.go +++ b/errors.go @@ -228,3 +228,47 @@ var ( ErrAssignmentNotFound = errors.New("cortex: assignment not found") ErrAssignmentUnavailable = errors.New("cortex: assignment transaction unavailable") ) + +// ErrInvalidDecision is returned when a checkpoint decision is malformed: +// its amended arguments (on the decision or on one call) are not a JSON +// object; it amends the whole decision's arguments on a suspension that +// holds more than one pending call; its Calls miss a pending call, name a +// call that is not pending, or sit beside a top-level amendment. It is also +// returned when a caller's own results carry what only a decision may set +// (edited arguments or the mark of a reviewer's denial). +var ErrInvalidDecision = errors.New("cortex: invalid checkpoint decision") + +// ErrDeferModelCall is what a model client returns, wrapped or as +// DeferredModelCallError, when it will not make a call right now but may later: +// the workspace is out of AI credits, say. The run pauses before that call +// with a checkpoint, nothing the run already did is lost, and an approved +// decision on the checkpoint continues the run by making the same call. +// Any other model error still fails the run. +var ErrDeferModelCall = errors.New("cortex: model call deferred") + +// DeferredModelCallError is ErrDeferModelCall with something to say: why the +// call waits (Reason, which becomes the checkpoint's reason) and what kind +// of wait it is (Kind, for example "credits", which the host reads back +// from the checkpoint's metadata). Return it from llm.Client, bare or +// wrapped; errors.Is matches it against ErrDeferModelCall. +type DeferredModelCallError struct { + // Reason is shown on the checkpoint. + Reason string + // Kind names what the call waits on. The engine does not interpret + // it. + Kind string +} + +// Error says the model call was deferred, and why. +func (d DeferredModelCallError) Error() string { + if d.Reason == "" { + return ErrDeferModelCall.Error() + } + return ErrDeferModelCall.Error() + ": " + d.Reason +} + +// Is reports a match for ErrDeferModelCall, so code that only knows the +// sentinel still recognizes a deferral. +func (d DeferredModelCallError) Is(target error) bool { + return target == ErrDeferModelCall +} diff --git a/extension/go.mod b/extension/go.mod index a97a7018..ea96a7e7 100644 --- a/extension/go.mod +++ b/extension/go.mod @@ -13,8 +13,8 @@ require ( github.com/xraph/cortex/sentinel v0.0.0-20261026122752-0b00c24b30ca github.com/xraph/forge v1.13.6 github.com/xraph/go-utils v1.5.0 - github.com/xraph/grove v1.10.2 - github.com/xraph/grove/drivers/sqlitedriver v1.10.2 + github.com/xraph/grove v1.10.3 + github.com/xraph/grove/drivers/sqlitedriver v1.10.3 github.com/xraph/nexus v1.9.3 github.com/xraph/shield v1.10.2 github.com/xraph/vessel v1.0.5 @@ -107,8 +107,8 @@ require ( github.com/xdg-go/scram v1.4.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/xraph/confy v1.1.0 // indirect - github.com/xraph/grove/drivers/mongodriver v1.10.2 // indirect - github.com/xraph/grove/drivers/pgdriver v1.10.2 // indirect + github.com/xraph/grove/drivers/mongodriver v1.10.3 // indirect + github.com/xraph/grove/drivers/pgdriver v1.10.3 // indirect github.com/xraph/sentinel v1.10.1 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect go.jetify.com/typeid/v2 v2.0.0-alpha.3 // indirect diff --git a/extension/go.sum b/extension/go.sum index 6be53450..efd8e522 100644 --- a/extension/go.sum +++ b/extension/go.sum @@ -378,14 +378,14 @@ github.com/xraph/forge v1.13.6 h1:JCLMd1eEd7Ay96iVg8tFlYDBMyVO8ltZSIMGRL7ktko= github.com/xraph/forge v1.13.6/go.mod h1:gg/KN2RcPOZl0DMxLRp7QQYBW3wLscXgYdUzfcpWDeI= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/mongodriver v1.10.2 h1:IrIqWzg72iL6qroh/llft+CH0FygMS2wyBwYkVeNQ2s= -github.com/xraph/grove/drivers/mongodriver v1.10.2/go.mod h1:x/t82VUmjCAuUluOHHfDrJCHdGKtd3M2pnKvBnUxvwU= -github.com/xraph/grove/drivers/pgdriver v1.10.2 h1:LMMqPMPtj1OgA1npfqOWsknMQZbJ7kFwMrdrj32Mp3I= -github.com/xraph/grove/drivers/pgdriver v1.10.2/go.mod h1:aV31nDRM4Igx4vqwjM9u8v0A1URL8YyDyXCuABLOuPA= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/mongodriver v1.10.3 h1:hk6xPlpDKeBFfp9MfFVdzValnITY5lG1ypseBwzIaJo= +github.com/xraph/grove/drivers/mongodriver v1.10.3/go.mod h1:x/t82VUmjCAuUluOHHfDrJCHdGKtd3M2pnKvBnUxvwU= +github.com/xraph/grove/drivers/pgdriver v1.10.3 h1:firu2F10BdN2ciaYmFw1MrQ4z0DKnNQz4ifg0gLzxes= +github.com/xraph/grove/drivers/pgdriver v1.10.3/go.mod h1:aV31nDRM4Igx4vqwjM9u8v0A1URL8YyDyXCuABLOuPA= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/xraph/nexus v1.9.3 h1:swjUBnkGFYs6f0sifyW7a7CmHetwjmslUKGsH19CFhQ= github.com/xraph/nexus v1.9.3/go.mod h1:0hU8ddSNwAw1bh7HlAe9v4ftR0VR0FlYZqpRjKAuhT0= github.com/xraph/sentinel v1.10.1 h1:zR/qLzkGgbTHOxqm4uxMmR7Ep4dAIq/890+BUA3kQec= diff --git a/go.mod b/go.mod index d363053f..f97a5f5d 100644 --- a/go.mod +++ b/go.mod @@ -10,10 +10,10 @@ require ( github.com/testcontainers/testcontainers-go/modules/mongodb v0.42.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 github.com/xraph/go-utils v1.5.0 - github.com/xraph/grove v1.10.2 - github.com/xraph/grove/drivers/mongodriver v1.10.2 - github.com/xraph/grove/drivers/pgdriver v1.10.2 - github.com/xraph/grove/drivers/sqlitedriver v1.10.2 + github.com/xraph/grove v1.10.3 + github.com/xraph/grove/drivers/mongodriver v1.10.3 + github.com/xraph/grove/drivers/pgdriver v1.10.3 + github.com/xraph/grove/drivers/sqlitedriver v1.10.3 github.com/xraph/nexus v1.9.3 github.com/xraph/shield v1.10.2 github.com/xraph/vessel v1.0.5 diff --git a/go.sum b/go.sum index 2782d977..0cb84821 100644 --- a/go.sum +++ b/go.sum @@ -144,14 +144,14 @@ github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/mongodriver v1.10.2 h1:IrIqWzg72iL6qroh/llft+CH0FygMS2wyBwYkVeNQ2s= -github.com/xraph/grove/drivers/mongodriver v1.10.2/go.mod h1:x/t82VUmjCAuUluOHHfDrJCHdGKtd3M2pnKvBnUxvwU= -github.com/xraph/grove/drivers/pgdriver v1.10.2 h1:LMMqPMPtj1OgA1npfqOWsknMQZbJ7kFwMrdrj32Mp3I= -github.com/xraph/grove/drivers/pgdriver v1.10.2/go.mod h1:aV31nDRM4Igx4vqwjM9u8v0A1URL8YyDyXCuABLOuPA= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/mongodriver v1.10.3 h1:hk6xPlpDKeBFfp9MfFVdzValnITY5lG1ypseBwzIaJo= +github.com/xraph/grove/drivers/mongodriver v1.10.3/go.mod h1:x/t82VUmjCAuUluOHHfDrJCHdGKtd3M2pnKvBnUxvwU= +github.com/xraph/grove/drivers/pgdriver v1.10.3 h1:firu2F10BdN2ciaYmFw1MrQ4z0DKnNQz4ifg0gLzxes= +github.com/xraph/grove/drivers/pgdriver v1.10.3/go.mod h1:aV31nDRM4Igx4vqwjM9u8v0A1URL8YyDyXCuABLOuPA= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/xraph/nexus v1.9.3 h1:swjUBnkGFYs6f0sifyW7a7CmHetwjmslUKGsH19CFhQ= github.com/xraph/nexus v1.9.3/go.mod h1:0hU8ddSNwAw1bh7HlAe9v4ftR0VR0FlYZqpRjKAuhT0= github.com/xraph/shield v1.10.2 h1:LJlnTeXvDov+GBPhOGsnyoiH/1Zsx5gaOPfNfSAGoxU= diff --git a/integrations/fabriq/go.mod b/integrations/fabriq/go.mod index f038dcaf..9061f6be 100644 --- a/integrations/fabriq/go.mod +++ b/integrations/fabriq/go.mod @@ -8,8 +8,8 @@ require ( github.com/xraph/cortex v1.8.1-0.20261025225257-d0eb82f6ed68 github.com/xraph/fabriq/core v1.10.2 github.com/xraph/go-utils v1.5.0 - github.com/xraph/grove v1.10.2 - github.com/xraph/grove/drivers/sqlitedriver v1.10.2 + github.com/xraph/grove v1.10.3 + github.com/xraph/grove/drivers/sqlitedriver v1.10.3 github.com/xraph/vessel v1.0.5 ) diff --git a/integrations/fabriq/go.sum b/integrations/fabriq/go.sum index 00b75d7b..893beeff 100644 --- a/integrations/fabriq/go.sum +++ b/integrations/fabriq/go.sum @@ -36,10 +36,10 @@ github.com/xraph/fabriq/core v1.10.2 h1:06c5vgMMPB9Dmeoqjca5E3vzuPoC8Q6V3gISN1HO github.com/xraph/fabriq/core v1.10.2/go.mod h1:GGSFmRNOHObOJLOwITpmgw8YlP3ylxNDmlR3eoW16FE= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/xraph/shield v1.10.2 h1:LJlnTeXvDov+GBPhOGsnyoiH/1Zsx5gaOPfNfSAGoxU= github.com/xraph/shield v1.10.2/go.mod h1:xkWUxiFghxQV74UVNfDsSz/mCjQ18mpt4W4kxGiBqJU= github.com/xraph/vessel v1.0.5 h1:4In3UmIJzOrym5+FXE/uvm1jR+gofUqVmEjKjjgE3gw= diff --git a/integrations/sandbox/go.mod b/integrations/sandbox/go.mod index 6fb7d31f..853dac03 100644 --- a/integrations/sandbox/go.mod +++ b/integrations/sandbox/go.mod @@ -10,8 +10,8 @@ require ( github.com/opencontainers/runtime-spec v1.3.0 github.com/stretchr/testify v1.12.1 github.com/xraph/cortex v1.8.0 - github.com/xraph/grove v1.10.2 - github.com/xraph/grove/drivers/sqlitedriver v1.10.2 + github.com/xraph/grove v1.10.3 + github.com/xraph/grove/drivers/sqlitedriver v1.10.3 golang.org/x/sys v0.49.0 modernc.org/sqlite v1.58.0 ) diff --git a/integrations/sandbox/go.sum b/integrations/sandbox/go.sum index b0c1d580..53e0629f 100644 --- a/integrations/sandbox/go.sum +++ b/integrations/sandbox/go.sum @@ -115,10 +115,10 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.jetify.com/typeid/v2 v2.0.0-alpha.3 h1:T6RPx6bNl10lp0JN2Xz/XcgLZWSlVmL58Xqy9cgTCcc= diff --git a/inttest/go.mod b/inttest/go.mod index 2237945e..69a5a469 100644 --- a/inttest/go.mod +++ b/inttest/go.mod @@ -15,7 +15,7 @@ require ( github.com/xraph/cortex/integrations/fabriq v0.0.0-20260827002006-bb2528ddd778 github.com/xraph/fabriq v1.10.2 github.com/xraph/fabriq/core v1.10.2 - github.com/xraph/grove/drivers/sqlitedriver v1.10.2 + github.com/xraph/grove/drivers/sqlitedriver v1.10.3 ) require ( @@ -118,10 +118,10 @@ require ( github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xraph/cortex/optimization v0.0.0-20261025141439-37b94d6cbe98 github.com/xraph/go-utils v1.5.0 // indirect - github.com/xraph/grove v1.10.2 - github.com/xraph/grove/drivers/pgdriver v1.10.2 // indirect - github.com/xraph/grove/kv v1.10.2 // indirect - github.com/xraph/grove/kv/drivers/redisdriver v1.10.2 // indirect + github.com/xraph/grove v1.10.3 + github.com/xraph/grove/drivers/pgdriver v1.10.3 // indirect + github.com/xraph/grove/kv v1.10.3 // indirect + github.com/xraph/grove/kv/drivers/redisdriver v1.10.3 // indirect github.com/xraph/trove v1.8.2 // indirect github.com/xraph/vessel v1.0.5 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect diff --git a/inttest/go.sum b/inttest/go.sum index ada20ae9..589efd6c 100644 --- a/inttest/go.sum +++ b/inttest/go.sum @@ -232,16 +232,16 @@ github.com/xraph/fabriq/core v1.10.2 h1:06c5vgMMPB9Dmeoqjca5E3vzuPoC8Q6V3gISN1HO github.com/xraph/fabriq/core v1.10.2/go.mod h1:GGSFmRNOHObOJLOwITpmgw8YlP3ylxNDmlR3eoW16FE= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/pgdriver v1.10.2 h1:LMMqPMPtj1OgA1npfqOWsknMQZbJ7kFwMrdrj32Mp3I= -github.com/xraph/grove/drivers/pgdriver v1.10.2/go.mod h1:aV31nDRM4Igx4vqwjM9u8v0A1URL8YyDyXCuABLOuPA= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= -github.com/xraph/grove/kv v1.10.2 h1:3ic9NWFSGLmcxmy+eHw9t0HrU4WCGnTsSdn56rabS1U= -github.com/xraph/grove/kv v1.10.2/go.mod h1:qW3j9k8OV14itMSQTLpC/INh35HL5ppWRknzUNas2/o= -github.com/xraph/grove/kv/drivers/redisdriver v1.10.2 h1:o82/hCBDJlFmu9ICozAcrnc8piObjN0W7rqTHX6MygE= -github.com/xraph/grove/kv/drivers/redisdriver v1.10.2/go.mod h1:4mAi3I+9Li2zNaBXa2TGUf9aGHlGAAvdbE6WdkfqyZg= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/pgdriver v1.10.3 h1:firu2F10BdN2ciaYmFw1MrQ4z0DKnNQz4ifg0gLzxes= +github.com/xraph/grove/drivers/pgdriver v1.10.3/go.mod h1:aV31nDRM4Igx4vqwjM9u8v0A1URL8YyDyXCuABLOuPA= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove/kv v1.10.3 h1:bnjKdhg/exUqg+YqdwlfT0Ud/Vp+THTHTAJwBTRG8GM= +github.com/xraph/grove/kv v1.10.3/go.mod h1:qW3j9k8OV14itMSQTLpC/INh35HL5ppWRknzUNas2/o= +github.com/xraph/grove/kv/drivers/redisdriver v1.10.3 h1:0dJYfizIMxM1lOgdFSAbf4sCLEN/+Fm4y9fFDnVsN7c= +github.com/xraph/grove/kv/drivers/redisdriver v1.10.3/go.mod h1:4mAi3I+9Li2zNaBXa2TGUf9aGHlGAAvdbE6WdkfqyZg= github.com/xraph/shield v1.10.2 h1:LJlnTeXvDov+GBPhOGsnyoiH/1Zsx5gaOPfNfSAGoxU= github.com/xraph/shield v1.10.2/go.mod h1:xkWUxiFghxQV74UVNfDsSz/mCjQ18mpt4W4kxGiBqJU= github.com/xraph/trove v1.8.2 h1:rPe55n9keHQgQ3iSmCyiv2JNfjjlE0yozqeARmeIz8Q= diff --git a/mcpbridge/go.sum b/mcpbridge/go.sum index 6570d7cc..f7fa7de1 100644 --- a/mcpbridge/go.sum +++ b/mcpbridge/go.sum @@ -35,10 +35,10 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/xraph/shield v1.10.2 h1:LJlnTeXvDov+GBPhOGsnyoiH/1Zsx5gaOPfNfSAGoxU= github.com/xraph/shield v1.10.2/go.mod h1:xkWUxiFghxQV74UVNfDsSz/mCjQ18mpt4W4kxGiBqJU= github.com/xraph/vessel v1.0.5 h1:4In3UmIJzOrym5+FXE/uvm1jR+gofUqVmEjKjjgE3gw= diff --git a/optimization/go.mod b/optimization/go.mod index f93c4d47..aee75f1e 100644 --- a/optimization/go.mod +++ b/optimization/go.mod @@ -6,10 +6,10 @@ require ( github.com/testcontainers/testcontainers-go/modules/mongodb v0.42.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 github.com/xraph/cortex v1.8.1-0.20261025225257-d0eb82f6ed68 - github.com/xraph/grove v1.10.2 - github.com/xraph/grove/drivers/mongodriver v1.10.2 - github.com/xraph/grove/drivers/pgdriver v1.10.2 - github.com/xraph/grove/drivers/sqlitedriver v1.10.2 + github.com/xraph/grove v1.10.3 + github.com/xraph/grove/drivers/mongodriver v1.10.3 + github.com/xraph/grove/drivers/pgdriver v1.10.3 + github.com/xraph/grove/drivers/sqlitedriver v1.10.3 go.jetify.com/typeid/v2 v2.0.0-alpha.3 go.mongodb.org/mongo-driver/v2 v2.10.0 ) diff --git a/optimization/go.sum b/optimization/go.sum index e610ee57..f6bd2b9c 100644 --- a/optimization/go.sum +++ b/optimization/go.sum @@ -139,14 +139,14 @@ github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/mongodriver v1.10.2 h1:IrIqWzg72iL6qroh/llft+CH0FygMS2wyBwYkVeNQ2s= -github.com/xraph/grove/drivers/mongodriver v1.10.2/go.mod h1:x/t82VUmjCAuUluOHHfDrJCHdGKtd3M2pnKvBnUxvwU= -github.com/xraph/grove/drivers/pgdriver v1.10.2 h1:LMMqPMPtj1OgA1npfqOWsknMQZbJ7kFwMrdrj32Mp3I= -github.com/xraph/grove/drivers/pgdriver v1.10.2/go.mod h1:aV31nDRM4Igx4vqwjM9u8v0A1URL8YyDyXCuABLOuPA= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/mongodriver v1.10.3 h1:hk6xPlpDKeBFfp9MfFVdzValnITY5lG1ypseBwzIaJo= +github.com/xraph/grove/drivers/mongodriver v1.10.3/go.mod h1:x/t82VUmjCAuUluOHHfDrJCHdGKtd3M2pnKvBnUxvwU= +github.com/xraph/grove/drivers/pgdriver v1.10.3 h1:firu2F10BdN2ciaYmFw1MrQ4z0DKnNQz4ifg0gLzxes= +github.com/xraph/grove/drivers/pgdriver v1.10.3/go.mod h1:aV31nDRM4Igx4vqwjM9u8v0A1URL8YyDyXCuABLOuPA= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/xraph/shield v1.10.2 h1:LJlnTeXvDov+GBPhOGsnyoiH/1Zsx5gaOPfNfSAGoxU= github.com/xraph/shield v1.10.2/go.mod h1:xkWUxiFghxQV74UVNfDsSz/mCjQ18mpt4W4kxGiBqJU= github.com/xraph/vessel v1.0.5 h1:4In3UmIJzOrym5+FXE/uvm1jR+gofUqVmEjKjjgE3gw= diff --git a/sentinel/go.mod b/sentinel/go.mod index 28329161..15dca542 100644 --- a/sentinel/go.mod +++ b/sentinel/go.mod @@ -7,8 +7,8 @@ replace github.com/xraph/cortex => ../ require ( github.com/xraph/cortex v1.8.1-0.20261025225257-d0eb82f6ed68 github.com/xraph/cortex/optimization v0.0.0-20261026160546-1b6f6bc9f367 - github.com/xraph/grove v1.10.2 - github.com/xraph/grove/drivers/sqlitedriver v1.10.2 + github.com/xraph/grove v1.10.3 + github.com/xraph/grove/drivers/sqlitedriver v1.10.3 github.com/xraph/sentinel v1.10.1 github.com/xraph/vessel v1.0.5 ) diff --git a/sentinel/go.sum b/sentinel/go.sum index c78ff8de..94509ba1 100644 --- a/sentinel/go.sum +++ b/sentinel/go.sum @@ -291,10 +291,10 @@ github.com/xraph/forge v1.13.6 h1:JCLMd1eEd7Ay96iVg8tFlYDBMyVO8ltZSIMGRL7ktko= github.com/xraph/forge v1.13.6/go.mod h1:gg/KN2RcPOZl0DMxLRp7QQYBW3wLscXgYdUzfcpWDeI= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/xraph/sentinel v1.10.1 h1:zR/qLzkGgbTHOxqm4uxMmR7Ep4dAIq/890+BUA3kQec= github.com/xraph/sentinel v1.10.1/go.mod h1:vuTrYQUVHVxCOK6JScuSp9cPfMpwNXYouXvIWZmVRQs= github.com/xraph/shield v1.10.2 h1:LJlnTeXvDov+GBPhOGsnyoiH/1Zsx5gaOPfNfSAGoxU= diff --git a/store/storetest/conformance.go b/store/storetest/conformance.go index 24db921c..73b3c991 100644 --- a/store/storetest/conformance.go +++ b/store/storetest/conformance.go @@ -8,8 +8,10 @@ package storetest import ( "context" + "encoding/json" "errors" "fmt" + "reflect" "strings" "sync" "testing" @@ -2052,6 +2054,107 @@ func testSameIdentifierCrossScope(t *testing.T, newStore func(t *testing.T) stor } }) + // A deferred model call pauses a run with no pending calls. The row + // reads back with its reason and an empty pending list, and a claim + // takes it like any other pause. + t.Run("SuspensionWithoutPendingCalls", func(t *testing.T) { + s := newStore(t) + ctx := ctxWithScope("ws_deferred_model_call") + r := mustPauseRun(t, s, ctx) + susp := newSuspension(r.ID) + susp.Reason = suspension.ReasonDeferredModelCall + susp.Pending = nil + if err := s.CreateSuspension(ctx, susp); err != nil { + t.Fatalf("create suspension: %v", err) + } + got, err := s.GetSuspension(ctx, r.ID) + if err != nil { + t.Fatalf("get suspension: %v", err) + } + if got.Reason != suspension.ReasonDeferredModelCall || len(got.Pending) != 0 { + t.Fatalf("suspension read back as (%q, %+v), want (%q, no pending calls)", got.Reason, got.Pending, suspension.ReasonDeferredModelCall) + } + if got.Cont.StepIndex != susp.Cont.StepIndex || len(got.Cont.Messages) != 1 { + t.Fatalf("continuation read back as %+v, want %+v", got.Cont, susp.Cont) + } + claimed, err := s.ClaimSuspension(ctx, r.ID) + if err != nil { + t.Fatalf("claim suspension: %v", err) + } + if claimed.ID != susp.ID || len(claimed.Pending) != 0 { + t.Fatalf("claimed %+v, want suspension %s with no pending calls", claimed, susp.ID) + } + }) + + // A decision's amended arguments are stored with it and read back + // the same JSON values (jsonb re-spaces them), whichever backend holds the checkpoint. + t.Run("CheckpointDecisionAmendedArguments", func(t *testing.T) { + s := newStore(t) + ctx := ctxWithScope("ws_amended_arguments") + r := mustCreateRun(t, s, ctx) + cp := mustCreateCheckpoint(t, s, ctx, r.ID, r.AgentID) + amended := json.RawMessage(`{"to":"b","cc":["c","d"],"n":2}`) + if err := s.Resolve(ctx, cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "reviewer", AmendedArguments: amended}); err != nil { + t.Fatalf("Resolve: %v", err) + } + got, err := s.GetCheckpoint(ctx, cp.ID) + if err != nil { + t.Fatalf("reload checkpoint: %v", err) + } + if got.Decision == nil { + t.Fatal("reloaded checkpoint has no decision") + } + var want, have map[string]any + if err = json.Unmarshal(amended, &want); err != nil { + t.Fatalf("decode want: %v", err) + } + if err = json.Unmarshal(got.Decision.AmendedArguments, &have); err != nil { + t.Fatalf("decode amended arguments %q: %v", got.Decision.AmendedArguments, err) + } + if !reflect.DeepEqual(want, have) { + t.Fatalf("amended arguments = %s, want %s", got.Decision.AmendedArguments, amended) + } + + // A per-call decision reads back call by call. + cp3 := mustCreateCheckpoint(t, s, ctx, r.ID, r.AgentID) + perCall := map[string]checkpoint.CallDecision{ + "call-1": {Approved: true, AmendedArguments: json.RawMessage(`{"to":"b"}`)}, + "call-2": {Reason: "wrong recipient"}, + } + if err = s.Resolve(ctx, cp3.ID, checkpoint.Decision{Approved: true, DecidedBy: "reviewer", Calls: perCall}); err != nil { + t.Fatalf("Resolve with calls: %v", err) + } + got3, err := s.GetCheckpoint(ctx, cp3.ID) + if err != nil || got3.Decision == nil { + t.Fatalf("reload third checkpoint: %v", err) + } + if len(got3.Decision.Calls) != 2 { + t.Fatalf("calls = %+v, want 2 entries", got3.Decision.Calls) + } + c1, c2 := got3.Decision.Calls["call-1"], got3.Decision.Calls["call-2"] + // jsonb re-spaces JSON, so compare the objects, not the bytes. + var c1Args map[string]any + if decodeErr := json.Unmarshal(c1.AmendedArguments, &c1Args); decodeErr != nil || !reflect.DeepEqual(c1Args, map[string]any{"to": "b"}) { + t.Fatalf("call-1 amended arguments = %s (%v), want {\"to\":\"b\"}", c1.AmendedArguments, decodeErr) + } + if !c1.Approved || c2.Approved || c2.Reason != "wrong recipient" || len(c2.AmendedArguments) != 0 { + t.Fatalf("calls read back as %+v, want %+v", got3.Decision.Calls, perCall) + } + + // A decision without them reads back without them. + cp2 := mustCreateCheckpoint(t, s, ctx, r.ID, r.AgentID) + if resolveErr := s.Resolve(ctx, cp2.ID, checkpoint.Decision{Approved: true}); resolveErr != nil { + t.Fatalf("Resolve without arguments: %v", resolveErr) + } + got2, err := s.GetCheckpoint(ctx, cp2.ID) + if err != nil || got2.Decision == nil { + t.Fatalf("reload second checkpoint: %v", err) + } + if len(got2.Decision.AmendedArguments) != 0 { + t.Fatalf("amended arguments = %s, want none", got2.Decision.AmendedArguments) + } + }) + // Agent is the one entity below that IS scope-guarded: the same name // reused across two scopes must resolve to two distinct rows, proving // UNIQUE (scope_canon, name), not the retired UNIQUE (app_id, name), diff --git a/suspension/suspension.go b/suspension/suspension.go index e4e3b393..a2695548 100644 --- a/suspension/suspension.go +++ b/suspension/suspension.go @@ -38,6 +38,12 @@ const ( // sent. The messaging bus resumes these, off its own correlation // ledger; nobody else may. ReasonAgentReply SuspendReason = "agent_reply" + // ReasonDeferredModelCall means the model client deferred the run's + // next model call (cortex.DeferredModelCallError). The suspension has no + // pending calls, its continuation is the state from before that call, + // and only a decision on its checkpoint continues it, by making the + // same call. + ReasonDeferredModelCall SuspendReason = "deferred_model_call" ) // PendingCall is one tool call awaiting a result from outside the engine. @@ -72,7 +78,7 @@ type Notice struct { // ExpiresAt is the suspension's deadline, nil when it has none. ExpiresAt *time.Time // CheckpointID is the checkpoint a person decides. It is zero unless - // Reason is ReasonApproval. + // Reason is ReasonApproval or ReasonDeferredModelCall. CheckpointID id.CheckpointID Scope cortex.Scope }