From a1e44bd93e80898c1300aec7ab5089cb03542efc Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Fri, 13 Nov 2026 02:45:29 -0600 Subject: [PATCH 1/9] feat(checkpoint): let a decision amend the suspended call's arguments An approval can now carry AmendedArguments. They replace the whole argument object of the one pending call, and ContinueRun runs the tool with them instead of the model's. The authorizer, the tool call row and the model's history all see the amended arguments, and the row records who amended them. Arguments that are not a JSON object, or an edit on a suspension with more than one pending call, are refused with ErrInvalidDecision. A rejection ignores the edit and does not record it. --- checkpoint/checkpoint.go | 9 +++ engine/continue.go | 32 +++++++- engine/continue_amend_test.go | 133 +++++++++++++++++++++++++++++++++ engine/continue_test.go | 2 + engine/engine.go | 6 +- engine/prepared_resume.go | 5 +- engine/resume.go | 60 ++++++++++++++- errors.go | 5 ++ store/storetest/conformance.go | 45 +++++++++++ 9 files changed, 288 insertions(+), 9 deletions(-) create mode 100644 engine/continue_amend_test.go diff --git a/checkpoint/checkpoint.go b/checkpoint/checkpoint.go index 46ee1184..2f5a62a6 100644 --- a/checkpoint/checkpoint.go +++ b/checkpoint/checkpoint.go @@ -2,6 +2,7 @@ package checkpoint import ( + "encoding/json" "time" "github.com/xraph/cortex" @@ -23,6 +24,14 @@ type Decision struct { // Anything that serves a checkpoint to a reader goes through // Checkpoint.Public, which drops it. DecidedByPrincipal []byte `json:"decided_by_principal,omitempty"` + // AmendedArguments, on an approval, are the arguments the suspended + // call runs with instead of the model's. They replace the whole + // argument object, nothing is merged. They must be a JSON object, and + // the suspension must hold exactly one pending call, or the decision + // is refused with cortex.ErrInvalidDecision. The engine does not + // validate them against the tool's schema; that is the caller's job. + // A rejection ignores them and does not record them. + AmendedArguments json.RawMessage `json:"amended_arguments,omitempty"` } // Checkpoint represents a point where a run pauses for approval. diff --git a/engine/continue.go b/engine/continue.go index 8964c2bb..599ad74b 100644 --- a/engine/continue.go +++ b/engine/continue.go @@ -91,7 +91,10 @@ func (e *Engine) decideCheckpoint(ctx context.Context, cpID id.CheckpointID, d c if !d.Approved { source = run.ReadyRejection } - in := decisionInput(susp.Pending, d) + in, err := decisionInput(susp.Pending, d) + if err != nil { + return nil, err + } if _, previewErr := e.previewResume(ctx, cp.RunID, in, resumeSourceApproval, susp, nil); previewErr != nil { return nil, previewErr } @@ -205,6 +208,12 @@ func (e *Engine) openDecision(ctx context.Context, cpID id.CheckpointID, d check return nil, d, fmt.Errorf("resolve checkpoint %s: %w", cpID, err) } d.DecidedByPrincipal = decidedBy + if err := checkArgumentsObject(d.AmendedArguments); err != nil { + return nil, d, fmt.Errorf("resolve checkpoint %s: %w", cpID, err) + } + if !d.Approved { + d.AmendedArguments = nil + } return cp, d, nil } @@ -292,16 +301,31 @@ func (e *Engine) recordedAs(ctx context.Context, cp *checkpoint.Checkpoint, d ch // approval asks the engine to execute each one, since the calls were // stopped before dispatch. A rejection answers each one with an error the // model reads. -func decisionInput(pending []suspension.PendingCall, d checkpoint.Decision) ResumeInput { +// +// Amended arguments belong to one call, so an approval that carries them +// is refused with cortex.ErrInvalidDecision unless exactly one call is +// pending. +func decisionInput(pending []suspension.PendingCall, d checkpoint.Decision) (ResumeInput, error) { in := ResumeInput{ToolResults: make([]ToolResult, 0, len(pending)), rejected: !d.Approved} + if d.Approved && len(d.AmendedArguments) > 0 && len(pending) != 1 { + return in, fmt.Errorf("%w: amended arguments need exactly one pending call, found %d", cortex.ErrInvalidDecision, len(pending)) + } for _, p := range pending { if d.Approved { - in.ToolResults = append(in.ToolResults, ToolResult{ToolCallID: p.ID, Execute: true}) + in.ToolResults = append(in.ToolResults, ToolResult{ToolCallID: p.ID, Execute: true, Arguments: d.AmendedArguments, AmendedBy: amendedBy(d)}) } else { in.ToolResults = append(in.ToolResults, ToolResult{ToolCallID: p.ID, Error: rejectionMessage(d)}) } } - return in + return in, nil +} + +// amendedBy names who edited a call's arguments, or "" when nobody did. +func amendedBy(d checkpoint.Decision) string { + if len(d.AmendedArguments) == 0 { + return "" + } + return d.DecidedBy } // rejectionMessage is the error a rejected call returns to the model. diff --git a/engine/continue_amend_test.go b/engine/continue_amend_test.go new file mode 100644 index 00000000..100bb611 --- /dev/null +++ b/engine/continue_amend_test.go @@ -0,0 +1,133 @@ +package engine + +import ( + "context" + "encoding/json" + "slices" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/xraph/cortex" + "github.com/xraph/cortex/checkpoint" + "github.com/xraph/cortex/llm" + "github.com/xraph/cortex/run" +) + +func (f *continueFixture) seenArguments() []string { + f.mu.Lock() + defer f.mu.Unlock() + return slices.Clone(f.args) +} + +func newAmendFixture(t *testing.T, opts ...Option) *continueFixture { + t.Helper() + ctx := cortexScopeContext() + return buildContinueFixture(ctx, t, newApprovalStore(ctx, t), []llm.ToolCall{{ID: "call-1", Name: gatedTool().Name, Arguments: `{"to":"a"}`}}, opts...) +} + +func cortexScopeContext() context.Context { + return cortex.WithScope(context.Background(), approvalScope()) +} + +func TestContinueRun_amendedArgumentsReplaceTheModelsArguments(t *testing.T) { + f := newAmendFixture(t) + paused, cp := f.pause(t) + + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{ + Approved: true, + DecidedBy: "bob", + AmendedArguments: json.RawMessage(`{"to":"b"}`), + })) + done, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, done.State) + + require.Equal(t, []string{`{"to":"b"}`}, f.seenArguments(), "the tool must run once, with the amended arguments") + + // The model sees one result for the call, and its own history shows + // the arguments that ran. + messages := f.model.lastRequest() + results := 0 + for _, m := range messages { + if m.Role == "tool" && m.ToolCallID == "call-1" { + results++ + } + for _, tc := range m.ToolCalls { + if tc.ID == "call-1" { + require.JSONEq(t, `{"to":"b"}`, tc.Arguments) + } + } + } + require.Equal(t, 1, results) + + // The decision keeps the edit, and the trace says who made it. + require.JSONEq(t, `{"to":"b"}`, string(f.checkpointState(t, cp.ID).Decision.AmendedArguments)) + steps, err := f.e.ListSteps(f.ctx, paused.ID) + require.NoError(t, err) + found := false + for _, step := range steps { + calls, listErr := f.e.ListToolCalls(f.ctx, step.ID) + require.NoError(t, listErr) + for _, call := range calls { + if call.ToolName == gatedTool().Name { + found = true + require.JSONEq(t, `{"to":"b"}`, call.Arguments) + require.Equal(t, true, call.Metadata["arguments_amended"]) + require.Equal(t, "bob", call.Metadata["amended_by"]) + } + } + } + require.True(t, found, "the approved call left no tool call row") +} + +func TestDecideCheckpoint_amendedArgumentsMustBeAnObject(t *testing.T) { + for _, bad := range []string{`[1]`, `"x"`, `null`, `{`} { + t.Run(bad, func(t *testing.T) { + f := newAmendFixture(t) + paused, cp := f.pause(t) + + err := f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{ + Approved: true, + AmendedArguments: json.RawMessage(bad), + }) + require.ErrorIs(t, err, cortex.ErrInvalidDecision) + + // The refusal leaves the checkpoint open and the run paused. + require.Equal(t, "pending", f.checkpointState(t, cp.ID).State) + require.Equal(t, run.StatePaused, f.runState(t, paused.ID).State) + }) + } +} + +func TestDecideCheckpoint_amendedArgumentsNeedOnePendingCall(t *testing.T) { + ctx := cortexScopeContext() + f := buildContinueFixture(ctx, t, newApprovalStore(ctx, t), []llm.ToolCall{ + {ID: "call-1", Name: gatedTool().Name, Arguments: `{"to":"a"}`}, + {ID: "call-2", Name: gatedTool().Name, Arguments: `{"to":"c"}`}, + }) + paused, cp := f.pause(t) + + err := f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Approved: true, AmendedArguments: json.RawMessage(`{"to":"b"}`)}) + require.ErrorIs(t, err, cortex.ErrInvalidDecision) + require.Equal(t, run.StatePaused, f.runState(t, paused.ID).State) +} + +func TestContinueRun_rejectedIgnoresAmendedArguments(t *testing.T) { + f := newAmendFixture(t, WithRejectionsReturnToModel()) + paused, cp := f.pause(t) + + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{ + DecidedBy: "bob", + Reason: "no", + AmendedArguments: json.RawMessage(`{"to":"b"}`), + })) + _, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + + require.Zero(t, f.ran.Load(), "a rejected call must never run") + content, ok := toolMessage(f.model.lastRequest()) + require.True(t, ok) + require.Contains(t, content, "rejected by bob: no") + require.Empty(t, f.checkpointState(t, cp.ID).Decision.AmendedArguments, "a rejection must not record an edit") +} diff --git a/engine/continue_test.go b/engine/continue_test.go index 91c196ce..0f91f483 100644 --- a/engine/continue_test.go +++ b/engine/continue_test.go @@ -136,6 +136,7 @@ type continueFixture struct { mu sync.Mutex ran atomic.Int32 principals []any + args []string } func newContinueFixture(t *testing.T, opts ...Option) *continueFixture { @@ -154,6 +155,7 @@ func buildContinueFixture(ctx context.Context, t *testing.T, st store.Store, fir f.ran.Add(1) f.mu.Lock() f.principals = append(f.principals, inv.Principal) + f.args = append(f.args, inv.Call.Arguments) f.mu.Unlock() return "deleted", nil }), diff --git a/engine/engine.go b/engine/engine.go index 422cfa7b..e8d7a15a 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -1064,7 +1064,11 @@ func (e *Engine) resumeDecided(ctx context.Context, cp *checkpoint.Checkpoint, s // resume rather than Resume: this is the one caller allowed to say a // checkpoint decided these calls, and it says so having just read a // pending checkpoint for this run. - if _, err := e.resume(ctx, cp.RunID, decisionInput(susp.Pending, decision), resumeSourceApproval); err != nil { + in, err := decisionInput(susp.Pending, decision) + if err != nil { + return err + } + if _, err := e.resume(ctx, cp.RunID, in, resumeSourceApproval); err != nil { return fmt.Errorf("resume decided run %s: %w", cp.RunID, err) } return nil diff --git a/engine/prepared_resume.go b/engine/prepared_resume.go index 85b43aae..6188ab1b 100644 --- a/engine/prepared_resume.go +++ b/engine/prepared_resume.go @@ -82,7 +82,8 @@ func (e *Engine) authorizeResumedCalls(ctx context.Context, r *run.Run, susp *su byID[result.ToolCallID] = result } for _, pending := range susp.Pending { - if !byID[pending.ID].Execute { + result := byID[pending.ID] + if !result.Execute { continue } if !susp.Cont.Config.Budget.Deadline.IsZero() && !time.Now().Before(susp.Cont.Config.Budget.Deadline) { @@ -91,7 +92,7 @@ func (e *Engine) authorizeResumedCalls(ctx context.Context, r *run.Run, susp *su if st.preparedSnapshot != nil && len(filterByName(st.preparedSnapshot.Tools, []string{pending.Name})) == 0 { return fmt.Errorf("cortex: resumed tool is outside prepared allowlist: %w", cortex.ErrInvalidContinuation) } - if err := e.authorizeResumedTool(ctx, subject, llm.ToolCall{ID: pending.ID, Name: pending.Name, Arguments: pending.Arguments}, susp.Reason == suspension.ReasonApproval); err != nil { + if err := e.authorizeResumedTool(ctx, subject, llm.ToolCall{ID: pending.ID, Name: pending.Name, Arguments: resumedArguments(pending, result)}, susp.Reason == suspension.ReasonApproval); err != nil { return err } } diff --git a/engine/resume.go b/engine/resume.go index 79fc9223..21a09e21 100644 --- a/engine/resume.go +++ b/engine/resume.go @@ -2,6 +2,7 @@ package engine import ( "context" + "encoding/json" "errors" "fmt" "reflect" @@ -58,6 +59,12 @@ type ToolResult struct { // external tool is answered by supplying its result, since the engine // is not the thing that runs it. Execute bool `json:"execute,omitempty"` + // Arguments, with Execute, replace the arguments the call was + // suspended with: a decider edited them. They must be a JSON object. + // Only an approval decision sets them. + Arguments json.RawMessage `json:"arguments,omitempty"` + // AmendedBy says who edited Arguments. It goes on the call's trace. + AmendedBy string `json:"amended_by,omitempty"` } // ResumeInput is everything a caller supplies to continue a suspended @@ -774,6 +781,10 @@ func (e *Engine) recordResumedCalls(ctx context.Context, ag *agent.Config, r *ru } } completedAt := time.Now().UTC() + if res.Execute && len(res.Arguments) > 0 { + // The model's history says what ran, not what it first asked for. + amendMessageArguments(st.messages, p.ID, string(res.Arguments)) + } if !stepID.IsNil() { toolCall := &run.ToolCall{ @@ -782,7 +793,7 @@ func (e *Engine) recordResumedCalls(ctx context.Context, ag *agent.Config, r *ru StepID: stepID, RunID: r.ID, ToolName: publicToolName(ctx, p.Name), - Arguments: publicGenerationText(r, p.Arguments), + Arguments: publicGenerationText(r, resumedArguments(p, res)), Result: publicGenerationText(r, content), Error: publicToolErrorText(ctx, errText), // The call really was outstanding from the moment the run @@ -791,6 +802,9 @@ func (e *Engine) recordResumedCalls(ctx context.Context, ag *agent.Config, r *ru StartedAt: &startedAt, CompletedAt: &completedAt, } + if res.Execute && len(res.Arguments) > 0 { + toolCall.Metadata = map[string]any{"arguments_amended": true, "amended_by": res.AmendedBy} + } if err := e.store.CreateToolCall(ctx, toolCall); err != nil { e.logError("create resumed tool call", err, log.String("run_id", r.ID.String())) } @@ -820,6 +834,29 @@ func (e *Engine) recordResumedCalls(ctx context.Context, ag *agent.Config, r *ru return nil } +// resumedArguments is what an approved call runs with: the arguments a +// decider amended, if they did, else the ones the model asked for. +func resumedArguments(p suspension.PendingCall, res ToolResult) string { + if res.Execute && len(res.Arguments) > 0 { + return string(res.Arguments) + } + return p.Arguments +} + +// checkArgumentsObject reports whether amended arguments, when there are +// any, are a JSON object. Validating them against the tool's schema is +// left to the caller. +func checkArgumentsObject(args json.RawMessage) error { + if len(args) == 0 { + return nil + } + var obj map[string]json.RawMessage + if err := json.Unmarshal(args, &obj); err != nil || obj == nil { + return fmt.Errorf("%w: amended arguments must be a JSON object", cortex.ErrInvalidDecision) + } + return nil +} + // resumedCallResult returns checked-call content, error text and host outcome. // It is also where an approved call is finally executed. // @@ -847,7 +884,7 @@ func (e *Engine) resumedCallResult(ctx context.Context, s cortex.Subject, p susp return jsonResult("error", err.Error()), err.Error(), outcomeFailed } - call := llm.ToolCall{ID: p.ID, Name: p.Name, Arguments: p.Arguments} + call := llm.ToolCall{ID: p.ID, Name: p.Name, Arguments: resumedArguments(p, res)} content, outcome, _ = e.traceTool(ctx, call, func(ctx context.Context) (string, toolOutcome, pendingWait) { out, callOutcome := e.runResumedTool(ctx, s, call, approved, &errText) return out, callOutcome, pendingWait{} @@ -977,6 +1014,12 @@ func validateResults(pending []suspension.PendingCall, results []ToolResult) err if !ok { return fmt.Errorf("%w: unexpected result for call id %q", cortex.ErrResultsMismatch, r.ToolCallID) } + if len(r.Arguments) > 0 && !r.Execute { + return fmt.Errorf("%w: arguments for call id %q without execute", cortex.ErrInvalidDecision, r.ToolCallID) + } + if err := checkArgumentsObject(r.Arguments); err != nil { + return err + } if n > 0 { return fmt.Errorf("%w: duplicate result for call id %q", cortex.ErrResultsMismatch, r.ToolCallID) } @@ -989,3 +1032,16 @@ func validateResults(pending []suspension.PendingCall, results []ToolResult) err } return nil } + +// amendMessageArguments rewrites the arguments of the assistant tool call +// callID in messages, so the conversation the model reads next matches +// the call that ran. +func amendMessageArguments(messages []llm.Message, callID, args string) { + for i := range messages { + for j := range messages[i].ToolCalls { + if messages[i].ToolCalls[j].ID == callID { + messages[i].ToolCalls[j].Arguments = args + } + } + } +} diff --git a/errors.go b/errors.go index 2f1f93a1..9cf1f3e7 100644 --- a/errors.go +++ b/errors.go @@ -228,3 +228,8 @@ var ( ErrAssignmentNotFound = errors.New("cortex: assignment not found") ErrAssignmentUnavailable = errors.New("cortex: assignment transaction unavailable") ) + +// ErrInvalidDecision is returned when a checkpoint decision is malformed: +// its amended arguments are not a JSON object, or it amends a suspension +// that holds more than one pending call. +var ErrInvalidDecision = errors.New("cortex: invalid checkpoint decision") diff --git a/store/storetest/conformance.go b/store/storetest/conformance.go index 24db921c..a69204c3 100644 --- a/store/storetest/conformance.go +++ b/store/storetest/conformance.go @@ -8,8 +8,10 @@ package storetest import ( "context" + "encoding/json" "errors" "fmt" + "reflect" "strings" "sync" "testing" @@ -2052,6 +2054,49 @@ func testSameIdentifierCrossScope(t *testing.T, newStore func(t *testing.T) stor } }) + // A decision's amended arguments are stored with it and read back + // byte for byte, whichever backend holds the checkpoint. + t.Run("CheckpointDecisionAmendedArguments", func(t *testing.T) { + s := newStore(t) + ctx := ctxWithScope("ws_amended_arguments") + r := mustCreateRun(t, s, ctx) + cp := mustCreateCheckpoint(t, s, ctx, r.ID, r.AgentID) + amended := json.RawMessage(`{"to":"b","cc":["c","d"],"n":2}`) + if err := s.Resolve(ctx, cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "reviewer", AmendedArguments: amended}); err != nil { + t.Fatalf("Resolve: %v", err) + } + got, err := s.GetCheckpoint(ctx, cp.ID) + if err != nil { + t.Fatalf("reload checkpoint: %v", err) + } + if got.Decision == nil { + t.Fatal("reloaded checkpoint has no decision") + } + var want, have map[string]any + if err := json.Unmarshal(amended, &want); err != nil { + t.Fatalf("decode want: %v", err) + } + if err := json.Unmarshal(got.Decision.AmendedArguments, &have); err != nil { + t.Fatalf("decode amended arguments %q: %v", got.Decision.AmendedArguments, err) + } + if !reflect.DeepEqual(want, have) { + t.Fatalf("amended arguments = %s, want %s", got.Decision.AmendedArguments, amended) + } + + // A decision without them reads back without them. + cp2 := mustCreateCheckpoint(t, s, ctx, r.ID, r.AgentID) + if err := s.Resolve(ctx, cp2.ID, checkpoint.Decision{Approved: true}); err != nil { + t.Fatalf("Resolve without arguments: %v", err) + } + got2, err := s.GetCheckpoint(ctx, cp2.ID) + if err != nil || got2.Decision == nil { + t.Fatalf("reload second checkpoint: %v", err) + } + if len(got2.Decision.AmendedArguments) != 0 { + t.Fatalf("amended arguments = %s, want none", got2.Decision.AmendedArguments) + } + }) + // Agent is the one entity below that IS scope-guarded: the same name // reused across two scopes must resolve to two distinct rows, proving // UNIQUE (scope_canon, name), not the retired UNIQUE (app_id, name), From 511e6dbab961a972597145a7d204f6b087f0e894 Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Fri, 13 Nov 2026 03:33:46 -0600 Subject: [PATCH 2/9] feat(checkpoint): decide each pending call on its own A decision can carry Calls, keyed by pending tool call id. Each call is approved, approved with edited arguments, or rejected with a reason the model reads. A pending call with no entry, an entry for a call that is not pending, or a non-object edit is refused with ErrInvalidDecision before anything moves. --- checkpoint/checkpoint.go | 22 +++++++ engine/continue.go | 96 +++++++++++++++++++++++++++-- engine/continue_amend_test.go | 106 +++++++++++++++++++++++++++++++++ engine/engine.go | 6 ++ engine/lease.go | 2 +- engine/resume.go | 6 +- store/storetest/conformance.go | 21 +++++++ 7 files changed, 252 insertions(+), 7 deletions(-) diff --git a/checkpoint/checkpoint.go b/checkpoint/checkpoint.go index 2f5a62a6..14e57f58 100644 --- a/checkpoint/checkpoint.go +++ b/checkpoint/checkpoint.go @@ -32,6 +32,28 @@ type Decision struct { // validate them against the tool's schema; that is the caller's job. // A rejection ignores them and does not record them. AmendedArguments json.RawMessage `json:"amended_arguments,omitempty"` + // Calls decides each pending tool call on its own, keyed by the + // pending call's id. When it is set, Approved is derived (true when + // any call is approved), AmendedArguments must be empty, and every + // pending call must have an entry and every entry must name a + // pending call, or the decision is refused with + // cortex.ErrInvalidDecision. A rejected call's Reason is what the + // model reads for that call. When it is empty, Approved and + // AmendedArguments decide every pending call together. + Calls map[string]CallDecision `json:"calls,omitempty"` +} + +// CallDecision is the outcome of one pending tool call within a Decision. +type CallDecision struct { + // Approved runs the call. A call that is not approved is rejected + // and answered with Reason. + Approved bool `json:"approved"` + // AmendedArguments, on an approved call, replace the whole argument + // object the call runs with. They must be a JSON object. A rejected + // call ignores them. + AmendedArguments json.RawMessage `json:"amended_arguments,omitempty"` + // Reason is what the model reads when the call is rejected. + Reason string `json:"reason,omitempty"` } // Checkpoint represents a point where a run pauses for approval. diff --git a/engine/continue.go b/engine/continue.go index 599ad74b..de11b447 100644 --- a/engine/continue.go +++ b/engine/continue.go @@ -6,6 +6,8 @@ import ( "encoding/json" "errors" "fmt" + "slices" + "strings" "time" "go.opentelemetry.io/otel/attribute" @@ -77,6 +79,9 @@ func (e *Engine) decideCheckpoint(ctx context.Context, cpID id.CheckpointID, d c if err != nil { return nil, err } + if err := checkCalls(susp.Pending, d); err != nil { + return nil, err + } if !d.Approved && !e.rejectionsToModel { if rejectErr := e.failRejected(ctx, cp, susp, d); rejectErr != nil { if errors.Is(rejectErr, cortex.ErrNotSuspended) && e.runMovedOn(ctx, cp) { @@ -211,6 +216,15 @@ func (e *Engine) openDecision(ctx context.Context, cpID id.CheckpointID, d check if err := checkArgumentsObject(d.AmendedArguments); err != nil { return nil, d, fmt.Errorf("resolve checkpoint %s: %w", cpID, err) } + if len(d.Calls) > 0 { + d = settleCalls(d) + for callID, cd := range d.Calls { + if err := checkArgumentsObject(cd.AmendedArguments); err != nil { + return nil, d, fmt.Errorf("resolve checkpoint %s: call %q: %w", cpID, callID, err) + } + } + return cp, d, nil + } if !d.Approved { d.AmendedArguments = nil } @@ -304,15 +318,31 @@ func (e *Engine) recordedAs(ctx context.Context, cp *checkpoint.Checkpoint, d ch // // Amended arguments belong to one call, so an approval that carries them // is refused with cortex.ErrInvalidDecision unless exactly one call is -// pending. +// pending. A decision with Calls decides each call on its own instead. func decisionInput(pending []suspension.PendingCall, d checkpoint.Decision) (ResumeInput, error) { in := ResumeInput{ToolResults: make([]ToolResult, 0, len(pending)), rejected: !d.Approved} + if len(d.Calls) > 0 { + if err := checkCalls(pending, d); err != nil { + return in, err + } + in.rejected = false + for _, p := range pending { + cd := d.Calls[p.ID] + if cd.Approved { + in.ToolResults = append(in.ToolResults, ToolResult{ToolCallID: p.ID, Execute: true, Arguments: cd.AmendedArguments, AmendedBy: amendedBy(d.DecidedBy, cd.AmendedArguments)}) + } else { + in.ToolResults = append(in.ToolResults, ToolResult{ToolCallID: p.ID, Error: rejectionMessage(checkpoint.Decision{DecidedBy: d.DecidedBy, Reason: cd.Reason}), Rejected: true}) + } + } + in.rejected = !d.Approved + return in, nil + } if d.Approved && len(d.AmendedArguments) > 0 && len(pending) != 1 { return in, fmt.Errorf("%w: amended arguments need exactly one pending call, found %d", cortex.ErrInvalidDecision, len(pending)) } for _, p := range pending { if d.Approved { - in.ToolResults = append(in.ToolResults, ToolResult{ToolCallID: p.ID, Execute: true, Arguments: d.AmendedArguments, AmendedBy: amendedBy(d)}) + in.ToolResults = append(in.ToolResults, ToolResult{ToolCallID: p.ID, Execute: true, Arguments: d.AmendedArguments, AmendedBy: amendedBy(d.DecidedBy, d.AmendedArguments)}) } else { in.ToolResults = append(in.ToolResults, ToolResult{ToolCallID: p.ID, Error: rejectionMessage(d)}) } @@ -320,12 +350,36 @@ func decisionInput(pending []suspension.PendingCall, d checkpoint.Decision) (Res return in, nil } +// checkCalls reports whether d.Calls names exactly the pending calls, once +// each, and nothing else. +func checkCalls(pending []suspension.PendingCall, d checkpoint.Decision) error { + if len(d.Calls) == 0 { + return nil + } + if len(d.AmendedArguments) > 0 { + return fmt.Errorf("%w: amend arguments per call in Calls, not on the decision", cortex.ErrInvalidDecision) + } + known := make(map[string]bool, len(pending)) + for _, p := range pending { + known[p.ID] = true + if _, ok := d.Calls[p.ID]; !ok { + return fmt.Errorf("%w: no decision for pending call %q", cortex.ErrInvalidDecision, p.ID) + } + } + for callID := range d.Calls { + if !known[callID] { + return fmt.Errorf("%w: call %q is not pending", cortex.ErrInvalidDecision, callID) + } + } + return nil +} + // amendedBy names who edited a call's arguments, or "" when nobody did. -func amendedBy(d checkpoint.Decision) string { - if len(d.AmendedArguments) == 0 { +func amendedBy(decidedBy string, amended json.RawMessage) string { + if len(amended) == 0 { return "" } - return d.DecidedBy + return decidedBy } // rejectionMessage is the error a rejected call returns to the model. @@ -756,3 +810,35 @@ func readyInput(ready *run.Readiness) (ResumeInput, resumeSource, error) { return in, resumeSourcePublic, fmt.Errorf("unknown ready source %q: %w", ready.Source, cortex.ErrInvalidContinuation) } } + +// settleCalls derives what a per-call decision means as a whole. It is +// approved when any call is, and a rejected call keeps no edit. When every +// call is rejected and the caller gave no reason, the reasons stand in for +// it, since a run that fails on the rejection reads the decision's Reason. +// d.Calls is copied, never changed in place. +func settleCalls(d checkpoint.Decision) checkpoint.Decision { + calls := make(map[string]checkpoint.CallDecision, len(d.Calls)) + ids := make([]string, 0, len(d.Calls)) + d.Approved = false + for callID, cd := range d.Calls { + if cd.Approved { + d.Approved = true + } else { + cd.AmendedArguments = nil + } + calls[callID] = cd + ids = append(ids, callID) + } + d.Calls = calls + if !d.Approved && d.Reason == "" { + slices.Sort(ids) + reasons := make([]string, 0, len(ids)) + for _, callID := range ids { + if r := calls[callID].Reason; r != "" { + reasons = append(reasons, r) + } + } + d.Reason = strings.Join(reasons, "; ") + } + return d +} diff --git a/engine/continue_amend_test.go b/engine/continue_amend_test.go index 100bb611..e5ab044e 100644 --- a/engine/continue_amend_test.go +++ b/engine/continue_amend_test.go @@ -131,3 +131,109 @@ func TestContinueRun_rejectedIgnoresAmendedArguments(t *testing.T) { require.Contains(t, content, "rejected by bob: no") require.Empty(t, f.checkpointState(t, cp.ID).Decision.AmendedArguments, "a rejection must not record an edit") } + +func newTwoCallFixture(t *testing.T) *continueFixture { + t.Helper() + ctx := cortexScopeContext() + return buildContinueFixture(ctx, t, newApprovalStore(ctx, t), []llm.ToolCall{ + {ID: "call-1", Name: gatedTool().Name, Arguments: `{"to":"a"}`}, + {ID: "call-2", Name: gatedTool().Name, Arguments: `{"to":"c"}`}, + }) +} + +func TestContinueRun_perCallDecisionsApproveAmendAndReject(t *testing.T) { + f := newTwoCallFixture(t) + paused, cp := f.pause(t) + + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{ + DecidedBy: "bob", + Calls: map[string]checkpoint.CallDecision{ + "call-1": {Approved: true, AmendedArguments: json.RawMessage(`{"to":"b"}`)}, + "call-2": {Reason: "wrong recipient"}, + }, + })) + done, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, done.State) + + require.Equal(t, []string{`{"to":"b"}`}, f.seenArguments(), "only the approved call runs, with the amended arguments") + var first, second string + for _, m := range f.model.lastRequest() { + if m.Role != "tool" { + continue + } + switch m.ToolCallID { + case "call-1": + first = m.Content + case "call-2": + second = m.Content + } + } + require.Equal(t, "deleted", first) + require.Contains(t, second, "rejected by bob: wrong recipient") + require.Equal(t, 1, countEvent(f.tools.snapshot(), "completed")) + require.Equal(t, 1, countEvent(f.tools.snapshot(), "denied")) + require.Zero(t, countEvent(f.tools.snapshot(), "failed")) + + decided := f.checkpointState(t, cp.ID).Decision + require.True(t, decided.Approved, "a decision with any approved call is approved") + require.Len(t, decided.Calls, 2) + require.JSONEq(t, `{"to":"b"}`, string(decided.Calls["call-1"].AmendedArguments)) + require.Equal(t, "wrong recipient", decided.Calls["call-2"].Reason) +} + +func TestDecideCheckpoint_perCallDecisionsMustNameEveryPendingCall(t *testing.T) { + cases := map[string]map[string]checkpoint.CallDecision{ + "missing call": {"call-1": {Approved: true}}, + "unknown call": {"call-1": {Approved: true}, "call-2": {Approved: true}, "call-9": {Approved: true}}, + "wrong call id": {"call-1": {Approved: true}, "call-9": {Approved: true}}, + "amend not obj": {"call-1": {Approved: true, AmendedArguments: json.RawMessage(`[1]`)}, "call-2": {Approved: true}}, + } + for name, calls := range cases { + t.Run(name, func(t *testing.T) { + f := newTwoCallFixture(t) + paused, cp := f.pause(t) + + err := f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Calls: calls}) + require.ErrorIs(t, err, cortex.ErrInvalidDecision) + require.Equal(t, "pending", f.checkpointState(t, cp.ID).State) + require.Equal(t, run.StatePaused, f.runState(t, paused.ID).State) + }) + } +} + +func TestDecideCheckpoint_perCallDecisionsRefuseTopLevelAmendment(t *testing.T) { + f := newTwoCallFixture(t) + _, cp := f.pause(t) + err := f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{ + AmendedArguments: json.RawMessage(`{"to":"b"}`), + Calls: map[string]checkpoint.CallDecision{"call-1": {Approved: true}, "call-2": {Approved: true}}, + }) + require.ErrorIs(t, err, cortex.ErrInvalidDecision) +} + +func TestDecideCheckpoint_perCallDecisionsAllRejectedFailsRunWithoutOption(t *testing.T) { + f := newTwoCallFixture(t) + paused, cp := f.pause(t) + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{ + DecidedBy: "bob", + Calls: map[string]checkpoint.CallDecision{"call-1": {Reason: "no"}, "call-2": {Reason: "no"}}, + })) + require.Equal(t, run.StateFailed, f.runState(t, paused.ID).State) + require.Zero(t, f.ran.Load()) + require.False(t, f.checkpointState(t, cp.ID).Decision.Approved) +} + +func TestResolveCheckpoint_perCallDecisionsContinueTheRun(t *testing.T) { + f := newTwoCallFixture(t) + paused, cp := f.pause(t) + require.NoError(t, f.e.ResolveCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{ + DecidedBy: "bob", + Calls: map[string]checkpoint.CallDecision{ + "call-1": {Reason: "no"}, + "call-2": {Approved: true, AmendedArguments: json.RawMessage(`{"to":"z"}`)}, + }, + })) + require.Equal(t, run.StateCompleted, f.runState(t, paused.ID).State) + require.Equal(t, []string{`{"to":"z"}`}, f.seenArguments()) +} diff --git a/engine/engine.go b/engine/engine.go index e8d7a15a..919f6924 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -939,6 +939,9 @@ func (e *Engine) ResolveCheckpoint(ctx context.Context, cpID id.CheckpointID, de // that cannot be read is treated as one that continues; decideCheckpoint // reports the read error itself. func (e *Engine) decisionContinues(ctx context.Context, cpID id.CheckpointID, d checkpoint.Decision) bool { + if len(d.Calls) > 0 { + d = settleCalls(d) + } if !d.Approved && !e.rejectionsToModel { return false } @@ -984,6 +987,9 @@ func (e *Engine) resolveInPlace(ctx context.Context, cpID id.CheckpointID, decis if bindErr != nil { return bindErr } + if err := checkCalls(susp.Pending, decision); err != nil { + return err + } if decision.Approved || e.rejectionsToModel { if err := e.resumeDecided(ctx, cp, susp, decision); err != nil { uncertain := errors.Is(err, cortex.ErrEffectUncertain) diff --git a/engine/lease.go b/engine/lease.go index a1267490..d5423104 100644 --- a/engine/lease.go +++ b/engine/lease.go @@ -308,7 +308,7 @@ func answeredCalls(pending []suspension.PendingCall, in ResumeInput) []suspensio continue } switch { - case res.Error != "" && in.rejected: + case res.Error != "" && (in.rejected || res.Rejected): done = append(done, suspension.ProgressResult{CallID: p.ID, Content: jsonResult("error", res.Error), Error: res.Error, Outcome: progressOutcome(outcomeDenied)}) case res.Error != "": done = append(done, suspension.ProgressResult{CallID: p.ID, Content: jsonResult("error", res.Error), Error: res.Error, Outcome: progressOutcome(outcomeFailed)}) diff --git a/engine/resume.go b/engine/resume.go index 21a09e21..61a194de 100644 --- a/engine/resume.go +++ b/engine/resume.go @@ -65,6 +65,10 @@ type ToolResult struct { Arguments json.RawMessage `json:"arguments,omitempty"` // AmendedBy says who edited Arguments. It goes on the call's trace. AmendedBy string `json:"amended_by,omitempty"` + // Rejected says this one call is a reviewer's rejection going back to + // the model, in a decision that approved other calls. It is recorded + // as denied rather than failed, as a whole-run rejection is. + Rejected bool `json:"rejected,omitempty"` } // ResumeInput is everything a caller supplies to continue a suspended @@ -815,7 +819,7 @@ func (e *Engine) recordResumedCalls(ctx context.Context, ag *agent.Config, r *ru source = verification.ToolLocal } switch { - case in.rejected && errText != "": + case (in.rejected || res.Rejected) && errText != "": outcome, source = outcomeDenied, verification.ToolLocal e.emitToolDenied(ctx, r.ID, p.Name, errText) case errText != "": diff --git a/store/storetest/conformance.go b/store/storetest/conformance.go index a69204c3..2c4e72d9 100644 --- a/store/storetest/conformance.go +++ b/store/storetest/conformance.go @@ -2083,6 +2083,27 @@ func testSameIdentifierCrossScope(t *testing.T, newStore func(t *testing.T) stor t.Fatalf("amended arguments = %s, want %s", got.Decision.AmendedArguments, amended) } + // A per-call decision reads back call by call. + cp3 := mustCreateCheckpoint(t, s, ctx, r.ID, r.AgentID) + perCall := map[string]checkpoint.CallDecision{ + "call-1": {Approved: true, AmendedArguments: json.RawMessage(`{"to":"b"}`)}, + "call-2": {Reason: "wrong recipient"}, + } + if err := s.Resolve(ctx, cp3.ID, checkpoint.Decision{Approved: true, DecidedBy: "reviewer", Calls: perCall}); err != nil { + t.Fatalf("Resolve with calls: %v", err) + } + got3, err := s.GetCheckpoint(ctx, cp3.ID) + if err != nil || got3.Decision == nil { + t.Fatalf("reload third checkpoint: %v", err) + } + if len(got3.Decision.Calls) != 2 { + t.Fatalf("calls = %+v, want 2 entries", got3.Decision.Calls) + } + c1, c2 := got3.Decision.Calls["call-1"], got3.Decision.Calls["call-2"] + if !c1.Approved || string(c1.AmendedArguments) != `{"to":"b"}` || c2.Approved || c2.Reason != "wrong recipient" || len(c2.AmendedArguments) != 0 { + t.Fatalf("calls read back as %+v, want %+v", got3.Decision.Calls, perCall) + } + // A decision without them reads back without them. cp2 := mustCreateCheckpoint(t, s, ctx, r.ID, r.AgentID) if err := s.Resolve(ctx, cp2.ID, checkpoint.Decision{Approved: true}); err != nil { From be0f0ade4375702b50f20a6925c812d7b7e39cf4 Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Fri, 13 Nov 2026 08:14:14 -0600 Subject: [PATCH 3/9] test(store): compare amended call arguments as JSON, not bytes postgres stores a decision as jsonb, which re-spaces the arguments. The conformance check now compares the decoded objects. --- store/storetest/conformance.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/store/storetest/conformance.go b/store/storetest/conformance.go index 2c4e72d9..1b5ef646 100644 --- a/store/storetest/conformance.go +++ b/store/storetest/conformance.go @@ -2100,7 +2100,12 @@ func testSameIdentifierCrossScope(t *testing.T, newStore func(t *testing.T) stor t.Fatalf("calls = %+v, want 2 entries", got3.Decision.Calls) } c1, c2 := got3.Decision.Calls["call-1"], got3.Decision.Calls["call-2"] - if !c1.Approved || string(c1.AmendedArguments) != `{"to":"b"}` || c2.Approved || c2.Reason != "wrong recipient" || len(c2.AmendedArguments) != 0 { + // jsonb re-spaces JSON, so compare the objects, not the bytes. + var c1Args map[string]any + if err := json.Unmarshal(c1.AmendedArguments, &c1Args); err != nil || !reflect.DeepEqual(c1Args, map[string]any{"to": "b"}) { + t.Fatalf("call-1 amended arguments = %s (%v), want {\"to\":\"b\"}", c1.AmendedArguments, err) + } + if !c1.Approved || c2.Approved || c2.Reason != "wrong recipient" || len(c2.AmendedArguments) != 0 { t.Fatalf("calls read back as %+v, want %+v", got3.Decision.Calls, perCall) } From d889ccbe362306b12db118bcd1d8673c97e6ba87 Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Fri, 13 Nov 2026 08:35:53 -0600 Subject: [PATCH 4/9] fix(checkpoint): treat a decision with different edits as a different decision A retry, or a racing decision, only counts as the same decision when its outcome, decider, edited arguments (compared as JSON) and per-call outcomes match what the run was made ready on. Anything else is refused as already decided, so the stored record always matches what ran. A lost record is rebuilt with its edits and per-call outcomes. A caller's own results can no longer set edited arguments or the mark of a reviewer's denial. --- engine/continue.go | 142 ++++++++++++++++++++-- engine/continue_amend_test.go | 208 +++++++++++++++++++++++++++++++++ engine/resume.go | 20 ++++ errors.go | 8 +- store/storetest/conformance.go | 2 +- 5 files changed, 366 insertions(+), 14 deletions(-) diff --git a/engine/continue.go b/engine/continue.go index de11b447..b8018235 100644 --- a/engine/continue.go +++ b/engine/continue.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "reflect" "slices" "strings" "time" @@ -109,7 +110,7 @@ func (e *Engine) decideCheckpoint(ctx context.Context, cpID id.CheckpointID, d c // that finds the run already ready on this decision did not. marked := err == nil if errors.Is(err, cortex.ErrNotSuspended) { - ready, err = e.retriedDecision(ctx, cp, source, err) + ready, err = e.retriedDecision(ctx, cp, source, in, d, err) } if err != nil { return nil, err @@ -251,7 +252,7 @@ func (e *Engine) recordDecision(ctx context.Context, cp *checkpoint.Checkpoint, // Anything else is the refusal MarkReady gave. A checkpoint decided in the // meantime is cortex.ErrInvalidState, as it would have been had this call // read it a moment later. -func (e *Engine) retriedDecision(ctx context.Context, cp *checkpoint.Checkpoint, source run.ReadySource, refused error) (*run.Run, error) { +func (e *Engine) retriedDecision(ctx context.Context, cp *checkpoint.Checkpoint, source run.ReadySource, in ResumeInput, d checkpoint.Decision, refused error) (*run.Run, error) { current, err := e.store.GetCheckpoint(ctx, cp.ID) if err != nil { return nil, fmt.Errorf("reload checkpoint %s: %w", cp.ID, err) @@ -279,6 +280,12 @@ func (e *Engine) retriedDecision(ctx context.Context, cp *checkpoint.Checkpoint, // checkpoint after the record. return nil, fmt.Errorf("resolve checkpoint %s: already decided %s: %w", cp.ID, r.Ready.Source, cortex.ErrInvalidState) } + if !sameReadyDecision(r.Ready, in, d) { + // The run is ready on this kind of decision, but not on this one: + // somebody else's edits or per-call outcomes are what will run, and + // they stand. A retry is the same decision, nothing less. + return nil, fmt.Errorf("resolve checkpoint %s: already decided differently: %w", cp.ID, cortex.ErrInvalidState) + } return r, nil } @@ -307,8 +314,79 @@ func (e *Engine) recordedAs(ctx context.Context, cp *checkpoint.Checkpoint, d ch if err != nil || current.Decision == nil { return false } - got := current.Decision - return got.Approved == d.Approved && got.DecidedBy == d.DecidedBy && bytes.Equal(got.DecidedByPrincipal, d.DecidedByPrincipal) + return sameDecision(*current.Decision, d) +} + +// sameDecision reports whether two decisions say the same thing by the +// same decider: the outcome, who decided, the edited arguments (as JSON, +// so spacing does not matter) and each call's outcome, edits and reason. +// A top-level decision and a Calls decision with one entry are the same +// shape, since a lost record is rebuilt from the run's results and cannot +// tell which of the two the decider used. +func sameDecision(a, b checkpoint.Decision) bool { + if a.Approved != b.Approved || a.DecidedBy != b.DecidedBy || !bytes.Equal(a.DecidedByPrincipal, b.DecidedByPrincipal) { + return false + } + a, b = flattenSingleCall(a), flattenSingleCall(b) + if !sameJSON(a.AmendedArguments, b.AmendedArguments) || len(a.Calls) != len(b.Calls) { + return false + } + for callID, ca := range a.Calls { + cb, ok := b.Calls[callID] + if !ok || ca.Approved != cb.Approved || ca.Reason != cb.Reason || !sameJSON(ca.AmendedArguments, cb.AmendedArguments) { + return false + } + } + return true +} + +// flattenSingleCall turns a Calls decision of one entry into the top-level +// form, whose outcome it already is. +func flattenSingleCall(d checkpoint.Decision) checkpoint.Decision { + if len(d.Calls) != 1 { + return d + } + for _, cd := range d.Calls { + d.Approved, d.AmendedArguments, d.Calls = cd.Approved, cd.AmendedArguments, nil + } + return d +} + +// sameReadyDecision reports whether the run was made ready on exactly the +// results in carries, by the same decider. A readiness that cannot be +// decoded is not the same. +func sameReadyDecision(ready *run.Readiness, in ResumeInput, d checkpoint.Decision) bool { + if ready.DecidedBy != d.DecidedBy || !bytes.Equal(ready.DecidedByPrincipal, d.DecidedByPrincipal) { + return false + } + var have []ToolResult + if err := json.Unmarshal(ready.Results, &have); err != nil || len(have) != len(in.ToolResults) { + return false + } + byID := make(map[string]ToolResult, len(have)) + for _, res := range have { + byID[res.ToolCallID] = res + } + for _, want := range in.ToolResults { + got, ok := byID[want.ToolCallID] + if !ok || got.Execute != want.Execute || got.Error != want.Error || got.Rejected != want.Rejected || got.AmendedBy != want.AmendedBy || !sameJSON(got.Arguments, want.Arguments) { + return false + } + } + return true +} + +// sameJSON reports whether two JSON documents decode to the same value. +// Two empty ones are the same. +func sameJSON(a, b json.RawMessage) bool { + if len(a) == 0 || len(b) == 0 { + return len(a) == len(b) + } + var va, vb any + if json.Unmarshal(a, &va) != nil || json.Unmarshal(b, &vb) != nil { + return bytes.Equal(a, b) + } + return reflect.DeepEqual(va, vb) } // decisionInput is what a decision hands back for each pending call. An @@ -318,25 +396,26 @@ func (e *Engine) recordedAs(ctx context.Context, cp *checkpoint.Checkpoint, d ch // // Amended arguments belong to one call, so an approval that carries them // is refused with cortex.ErrInvalidDecision unless exactly one call is -// pending. A decision with Calls decides each call on its own instead. +// pending. A decision with Calls decides each call on its own instead; its +// callers have already run checkCalls on it, and it needs no other +// preparation, since the outcome of each call is read from that call. func decisionInput(pending []suspension.PendingCall, d checkpoint.Decision) (ResumeInput, error) { - in := ResumeInput{ToolResults: make([]ToolResult, 0, len(pending)), rejected: !d.Approved} + in := ResumeInput{ToolResults: make([]ToolResult, 0, len(pending))} if len(d.Calls) > 0 { - if err := checkCalls(pending, d); err != nil { - return in, err - } - in.rejected = false + rejected := true for _, p := range pending { cd := d.Calls[p.ID] if cd.Approved { + rejected = false in.ToolResults = append(in.ToolResults, ToolResult{ToolCallID: p.ID, Execute: true, Arguments: cd.AmendedArguments, AmendedBy: amendedBy(d.DecidedBy, cd.AmendedArguments)}) } else { in.ToolResults = append(in.ToolResults, ToolResult{ToolCallID: p.ID, Error: rejectionMessage(checkpoint.Decision{DecidedBy: d.DecidedBy, Reason: cd.Reason}), Rejected: true}) } } - in.rejected = !d.Approved + in.rejected = rejected return in, nil } + in.rejected = !d.Approved if d.Approved && len(d.AmendedArguments) > 0 && len(pending) != 1 { return in, fmt.Errorf("%w: amended arguments need exactly one pending call, found %d", cortex.ErrInvalidDecision, len(pending)) } @@ -413,6 +492,9 @@ func (e *Engine) ProvideResults(ctx context.Context, runID id.AgentRunID, in Res if _, err := cortex.RequireScope(ctx); err != nil { return err } + if err := refuseDecisionFields(in.ToolResults); err != nil { + return err + } in = ResumeInput{ToolResults: in.ToolResults} susp, err := e.store.GetSuspension(ctx, runID) if err != nil { @@ -769,6 +851,7 @@ func (e *Engine) recordLostDecision(ctx context.Context, ready *run.Readiness) { Reason: "recorded when the run continued", DecidedAt: ready.At, } + d.AmendedArguments, d.Calls = decisionFromResults(ready) if d.DecidedBy == "" && len(d.DecidedByPrincipal) == 0 { d.DecidedBy = checkpointDecidedByEngine d.Reason = "the run was continued on this decision, but who made it was not recorded" @@ -842,3 +925,40 @@ func settleCalls(d checkpoint.Decision) checkpoint.Decision { } return d } + +// decisionFromResults rebuilds what a decision edited and decided per call +// from the results its run was made ready with, for a record that was lost. +// A single call is recorded the top-level way, several are recorded in +// Calls, where a rejection keeps the reason the model was given. +func decisionFromResults(ready *run.Readiness) (json.RawMessage, map[string]checkpoint.CallDecision) { + var results []ToolResult + if json.Unmarshal(ready.Results, &results) != nil { + return nil, nil + } + if len(results) == 1 && !results[0].Rejected { + return results[0].Arguments, nil + } + perCall := false + for _, res := range results { + if res.Rejected || (res.Execute && len(res.Arguments) > 0) { + perCall = true + } + } + if !perCall { + return nil, nil + } + by := ready.DecidedBy + if by == "" { + by = "a reviewer" + } + prefix := "rejected by " + by + ": " + calls := make(map[string]checkpoint.CallDecision, len(results)) + for _, res := range results { + if res.Execute { + calls[res.ToolCallID] = checkpoint.CallDecision{Approved: true, AmendedArguments: res.Arguments} + continue + } + calls[res.ToolCallID] = checkpoint.CallDecision{Reason: strings.TrimPrefix(res.Error, prefix)} + } + return nil, calls +} diff --git a/engine/continue_amend_test.go b/engine/continue_amend_test.go index e5ab044e..7e9b4983 100644 --- a/engine/continue_amend_test.go +++ b/engine/continue_amend_test.go @@ -3,7 +3,9 @@ package engine import ( "context" "encoding/json" + "fmt" "slices" + "sync" "testing" "github.com/stretchr/testify/require" @@ -12,6 +14,7 @@ import ( "github.com/xraph/cortex/checkpoint" "github.com/xraph/cortex/llm" "github.com/xraph/cortex/run" + "github.com/xraph/cortex/suspension" ) func (f *continueFixture) seenArguments() []string { @@ -237,3 +240,208 @@ func TestResolveCheckpoint_perCallDecisionsContinueTheRun(t *testing.T) { require.Equal(t, run.StateCompleted, f.runState(t, paused.ID).State) require.Equal(t, []string{`{"to":"z"}`}, f.seenArguments()) } + +func newFlakyTwoCallFixture(t *testing.T, opts ...Option) (*continueFixture, *flakyResolveStore) { + t.Helper() + ctx := cortexScopeContext() + st := &flakyResolveStore{Store: newApprovalStore(ctx, t)} + return buildContinueFixture(ctx, t, st, []llm.ToolCall{ + {ID: "call-1", Name: gatedTool().Name, Arguments: `{"to":"a"}`}, + {ID: "call-2", Name: gatedTool().Name, Arguments: `{"to":"c"}`}, + }, opts...), st +} + +func TestDecideCheckpoint_aRetryWithDifferentEditsIsRefused(t *testing.T) { + ctx := cortexScopeContext() + st := &flakyResolveStore{Store: newApprovalStore(ctx, t)} + f := buildContinueFixture(ctx, t, st, []llm.ToolCall{{ID: "call-1", Name: gatedTool().Name, Arguments: `{"to":"a"}`}}) + paused, cp := f.pause(t) + first := checkpoint.Decision{Approved: true, DecidedBy: "bob", AmendedArguments: json.RawMessage(`{"to":"x"}`)} + + // The first decision marks the run ready and loses its record. + st.fail.Store(true) + require.Error(t, f.e.DecideCheckpoint(as(ctx, bob), cp.ID, first)) + require.Equal(t, run.StateReady, f.runState(t, paused.ID).State) + + // Different edits, or none, are not a retry of it. + for _, other := range []checkpoint.Decision{ + {Approved: true, DecidedBy: "bob", AmendedArguments: json.RawMessage(`{"to":"y"}`)}, + {Approved: true, DecidedBy: "bob"}, + } { + err := f.e.DecideCheckpoint(as(ctx, bob), cp.ID, other) + require.ErrorIs(t, err, cortex.ErrInvalidState) + require.Equal(t, checkpointStatePending, f.checkpointState(t, cp.ID).State) + } + + // The same edits with different spacing are the same decision. + same := first + same.AmendedArguments = json.RawMessage(`{ "to": "x" }`) + require.NoError(t, f.e.DecideCheckpoint(as(ctx, bob), cp.ID, same)) + _, err := f.e.ContinueRun(ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, []string{`{"to":"x"}`}, f.seenArguments()) + require.JSONEq(t, `{"to":"x"}`, string(f.checkpointState(t, cp.ID).Decision.AmendedArguments)) +} + +func TestDecideCheckpoint_aRetryWithDifferentCallOutcomesIsRefused(t *testing.T) { + f, st := newFlakyTwoCallFixture(t) + paused, cp := f.pause(t) + first := checkpoint.Decision{DecidedBy: "bob", Calls: map[string]checkpoint.CallDecision{ + "call-1": {Approved: true}, + "call-2": {Reason: "no"}, + }} + st.fail.Store(true) + require.Error(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, first)) + + for name, calls := range map[string]map[string]checkpoint.CallDecision{ + "swapped": {"call-1": {Reason: "no"}, "call-2": {Approved: true}}, + "other reason": {"call-1": {Approved: true}, "call-2": {Reason: "later"}}, + "edit added": {"call-1": {Approved: true, AmendedArguments: json.RawMessage(`{"to":"b"}`)}, "call-2": {Reason: "no"}}, + "both approved": {"call-1": {Approved: true}, "call-2": {Approved: true}}, + } { + t.Run(name, func(t *testing.T) { + err := f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{DecidedBy: "bob", Calls: calls}) + require.ErrorIs(t, err, cortex.ErrInvalidState) + }) + } + + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, first)) + _, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, []string{`{"to":"a"}`}, f.seenArguments()) +} + +func TestDecideCheckpoint_concurrentDecisionsWithDifferentEditsStoreWhatRan(t *testing.T) { + for i := range 4 { + t.Run(fmt.Sprintf("round %d", i), func(t *testing.T) { + f := newAmendFixture(t) + paused, cp := f.pause(t) + + errs := make([]error, 2) + var wg sync.WaitGroup + for j, args := range []string{`{"to":"x"}`, `{"to":"y"}`} { + wg.Go(func() { + errs[j] = f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{ + Approved: true, DecidedBy: fmt.Sprintf("op-%d", j), AmendedArguments: json.RawMessage(args), + }) + }) + } + wg.Wait() + wins := 0 + for _, err := range errs { + if err == nil { + wins++ + continue + } + require.ErrorIs(t, err, cortex.ErrInvalidState) + } + require.Equal(t, 1, wins, "exactly one decision wins: %v", errs) + + _, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + ran := f.seenArguments() + require.Len(t, ran, 1) + require.JSONEq(t, ran[0], string(f.checkpointState(t, cp.ID).Decision.AmendedArguments), "the record must match what ran") + }) + } +} + +func TestContinueRun_aLostDecisionRecordKeepsEditsAndCallOutcomes(t *testing.T) { + t.Run("amended single call", func(t *testing.T) { + ctx := cortexScopeContext() + st := &flakyResolveStore{Store: newApprovalStore(ctx, t)} + f := buildContinueFixture(ctx, t, st, []llm.ToolCall{{ID: "call-1", Name: gatedTool().Name, Arguments: `{"to":"a"}`}}) + paused, cp := f.pause(t) + st.fail.Store(true) + require.Error(t, f.e.DecideCheckpoint(as(ctx, bob), cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "bob", AmendedArguments: json.RawMessage(`{"to":"x"}`)})) + _, err := f.e.ContinueRun(ctx, paused.ID) + require.NoError(t, err) + + got := f.checkpointState(t, cp.ID).Decision + require.True(t, got.Approved) + require.JSONEq(t, `{"to":"x"}`, string(got.AmendedArguments)) + }) + t.Run("per call", func(t *testing.T) { + f, st := newFlakyTwoCallFixture(t) + paused, cp := f.pause(t) + st.fail.Store(true) + require.Error(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{DecidedBy: "bob", Calls: map[string]checkpoint.CallDecision{ + "call-1": {Approved: true, AmendedArguments: json.RawMessage(`{"to":"x"}`)}, + "call-2": {Reason: "wrong recipient"}, + }})) + _, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + + got := f.checkpointState(t, cp.ID).Decision + require.True(t, got.Approved) + require.Len(t, got.Calls, 2) + require.True(t, got.Calls["call-1"].Approved) + require.JSONEq(t, `{"to":"x"}`, string(got.Calls["call-1"].AmendedArguments)) + require.False(t, got.Calls["call-2"].Approved) + require.Equal(t, "wrong recipient", got.Calls["call-2"].Reason) + }) +} + +func TestDecisionInput_doesNotNeedASettledDecision(t *testing.T) { + pending := []suspension.PendingCall{{ID: "call-1"}, {ID: "call-2"}} + in, err := decisionInput(pending, checkpoint.Decision{DecidedBy: "bob", Calls: map[string]checkpoint.CallDecision{ + "call-1": {Approved: true}, + "call-2": {Reason: "no"}, + }}) + require.NoError(t, err) + require.False(t, in.rejected, "one call is approved, so the decision is not a whole-run rejection") + require.True(t, in.ToolResults[0].Execute) + require.False(t, in.ToolResults[1].Execute) + + in, err = decisionInput(pending, checkpoint.Decision{Calls: map[string]checkpoint.CallDecision{"call-1": {}, "call-2": {}}}) + require.NoError(t, err) + require.True(t, in.rejected) +} + +func TestPublicResumeCannotMarkADenialOrAmendArguments(t *testing.T) { + ctx := cortexScopeContext() + f := buildContinueFixture(ctx, t, newApprovalStore(ctx, t), []llm.ToolCall{{ID: "call-1", Name: externalTool().Name, Arguments: `{}`}}) + paused, _ := f.pause(t) + for name, res := range map[string]ToolResult{ + "rejected": {ToolCallID: "call-1", Error: "boom", Rejected: true}, + "arguments": {ToolCallID: "call-1", Content: "x", Arguments: json.RawMessage(`{"a":1}`)}, + "amendedby": {ToolCallID: "call-1", Content: "x", AmendedBy: "mallory"}, + } { + t.Run(name, func(t *testing.T) { + in := ResumeInput{ToolResults: []ToolResult{res}} + require.ErrorIs(t, f.e.ProvideResults(ctx, paused.ID, in), cortex.ErrInvalidDecision) + _, err := f.e.Resume(ctx, paused.ID, in) + require.ErrorIs(t, err, cortex.ErrInvalidDecision) + require.Equal(t, run.StatePaused, f.runState(t, paused.ID).State) + }) + } +} + +func TestContinueRun_perCallAllRejectedReturnsEveryReasonToTheModel(t *testing.T) { + ctx := cortexScopeContext() + f := buildContinueFixture(ctx, t, newApprovalStore(ctx, t), []llm.ToolCall{ + {ID: "call-1", Name: gatedTool().Name, Arguments: `{"to":"a"}`}, + {ID: "call-2", Name: gatedTool().Name, Arguments: `{"to":"c"}`}, + }, WithRejectionsReturnToModel()) + paused, cp := f.pause(t) + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{DecidedBy: "bob", Calls: map[string]checkpoint.CallDecision{ + "call-1": {Reason: "first reason"}, + "call-2": {Reason: "second reason"}, + }})) + require.Equal(t, run.StateReady, f.runState(t, paused.ID).State) + done, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, done.State) + + seen := map[string]string{} + for _, m := range f.model.lastRequest() { + if m.Role == "tool" { + seen[m.ToolCallID] = m.Content + } + } + require.Contains(t, seen["call-1"], "rejected by bob: first reason") + require.Contains(t, seen["call-2"], "rejected by bob: second reason") + require.Zero(t, f.ran.Load()) + require.Equal(t, 2, countEvent(f.tools.snapshot(), "denied")) + require.Zero(t, countEvent(f.tools.snapshot(), "failed")) +} diff --git a/engine/resume.go b/engine/resume.go index 61a194de..9496257b 100644 --- a/engine/resume.go +++ b/engine/resume.go @@ -110,6 +110,9 @@ type resumption struct { // or a rebuilt message list are three different ways to corrupt a run // that looked fine at the call site and went wrong several turns later. func (e *Engine) Resume(ctx context.Context, runID id.AgentRunID, in ResumeInput) (*run.Run, error) { + if err := refuseDecisionFields(in.ToolResults); err != nil { + return nil, err + } return e.resume(ctx, runID, in, resumeSourcePublic) } @@ -186,6 +189,10 @@ func (e *Engine) resumeRun(ctx context.Context, runID id.AgentRunID, in ResumeIn // when execution completes, same as StreamAgent's, and the run's admission // slot is held until then. func (e *Engine) ResumeStream(ctx context.Context, runID id.AgentRunID, in ResumeInput, events chan<- StreamEvent) (err error) { + if err := refuseDecisionFields(in.ToolResults); err != nil { + close(events) + return err + } ctx, done, err := e.admit(ctx) if err != nil { close(events) @@ -847,6 +854,19 @@ func resumedArguments(p suspension.PendingCall, res ToolResult) string { return p.Arguments } +// refuseDecisionFields refuses results that carry what only a checkpoint +// decision may set: edited arguments, who edited them, and the mark of a +// reviewer's denial. A public caller must not be able to have an ordinary +// tool error recorded and emitted as a person's rejection. +func refuseDecisionFields(results []ToolResult) error { + for _, r := range results { + if r.Rejected || r.AmendedBy != "" || len(r.Arguments) > 0 { + return fmt.Errorf("%w: arguments and denials are set by a checkpoint decision, not by a caller's results", cortex.ErrInvalidDecision) + } + } + return nil +} + // checkArgumentsObject reports whether amended arguments, when there are // any, are a JSON object. Validating them against the tool's schema is // left to the caller. diff --git a/errors.go b/errors.go index 9cf1f3e7..9aaf443d 100644 --- a/errors.go +++ b/errors.go @@ -230,6 +230,10 @@ var ( ) // ErrInvalidDecision is returned when a checkpoint decision is malformed: -// its amended arguments are not a JSON object, or it amends a suspension -// that holds more than one pending call. +// its amended arguments (on the decision or on one call) are not a JSON +// object; it amends the whole decision's arguments on a suspension that +// holds more than one pending call; its Calls miss a pending call, name a +// call that is not pending, or sit beside a top-level amendment. It is also +// returned when a caller's own results carry what only a decision may set +// (edited arguments or the mark of a reviewer's denial). var ErrInvalidDecision = errors.New("cortex: invalid checkpoint decision") diff --git a/store/storetest/conformance.go b/store/storetest/conformance.go index 1b5ef646..81207b73 100644 --- a/store/storetest/conformance.go +++ b/store/storetest/conformance.go @@ -2055,7 +2055,7 @@ func testSameIdentifierCrossScope(t *testing.T, newStore func(t *testing.T) stor }) // A decision's amended arguments are stored with it and read back - // byte for byte, whichever backend holds the checkpoint. + // the same JSON values (jsonb re-spaces them), whichever backend holds the checkpoint. t.Run("CheckpointDecisionAmendedArguments", func(t *testing.T) { s := newStore(t) ctx := ctxWithScope("ws_amended_arguments") From 6680df74a202be9f5d24439a15ee0da1606b7499 Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Fri, 13 Nov 2026 08:59:27 -0600 Subject: [PATCH 5/9] feat(engine): suspend before a model call the client defers A model client can now return cortex.DeferredModelCall (or wrap cortex.ErrDeferModelCall) when it will not make a call yet, for example when the workspace is out of AI credits. The run pauses before that call with a deferred_model_call checkpoint that carries the reason and kind. Approving it continues the run with the same call, and nothing already run is repeated or counted twice. Rejecting it stops the run as cancelled with the decision's reason. Per-call decisions and amended arguments on it are refused with ErrInvalidDecision. --- checkpoint/checkpoint.go | 17 + engine/continue.go | 20 +- engine/deferred_model_call_lease_test.go | 97 ++++++ engine/deferred_model_call_test.go | 421 +++++++++++++++++++++++ engine/engine.go | 71 +++- engine/react.go | 47 +++ engine/resume.go | 14 + engine/suspend.go | 81 ++++- engine/sweeper.go | 3 + engine/telemetry.go | 2 +- errors.go | 35 ++ store/storetest/conformance.go | 32 ++ suspension/suspension.go | 8 +- 13 files changed, 834 insertions(+), 14 deletions(-) create mode 100644 engine/deferred_model_call_lease_test.go create mode 100644 engine/deferred_model_call_test.go diff --git a/checkpoint/checkpoint.go b/checkpoint/checkpoint.go index 14e57f58..59e4f276 100644 --- a/checkpoint/checkpoint.go +++ b/checkpoint/checkpoint.go @@ -9,6 +9,23 @@ import ( "github.com/xraph/cortex/id" ) +// MetadataKind is the metadata key that names what a checkpoint the engine +// opened is waiting on. It is set to KindDeferredModelCall on a checkpoint +// opened for a deferred model call, and absent on a tool approval. +const MetadataKind = "kind" + +// KindDeferredModelCall is the MetadataKind of a checkpoint opened because +// the model client deferred a call (cortex.DeferredModelCall). Such a +// checkpoint has no pending tool calls. An approval continues the run with +// the same model call; a rejection stops the run with the decision's +// reason. A decision that carries Calls or AmendedArguments is refused +// with cortex.ErrInvalidDecision. +const KindDeferredModelCall = "deferred_model_call" + +// MetadataDeferralKind is the metadata key that carries +// cortex.DeferredModelCall.Kind on a deferred model call's checkpoint. +const MetadataDeferralKind = "deferral_kind" + // Decision represents the resolution of a checkpoint. type Decision struct { Approved bool `json:"approved"` diff --git a/engine/continue.go b/engine/continue.go index b8018235..9bd5d0be 100644 --- a/engine/continue.go +++ b/engine/continue.go @@ -80,11 +80,11 @@ func (e *Engine) decideCheckpoint(ctx context.Context, cpID id.CheckpointID, d c if err != nil { return nil, err } - if err := checkCalls(susp.Pending, d); err != nil { + if err := checkDecision(susp, d); err != nil { return nil, err } - if !d.Approved && !e.rejectionsToModel { - if rejectErr := e.failRejected(ctx, cp, susp, d); rejectErr != nil { + if !d.Approved && (!e.rejectionsToModel || isDeferredModelCall(susp)) { + if rejectErr := e.endRejected(ctx, cp, susp, d); rejectErr != nil { if errors.Is(rejectErr, cortex.ErrNotSuspended) && e.runMovedOn(ctx, cp) { return nil, fmt.Errorf("resolve checkpoint %s: already decided: %w", cp.ID, cortex.ErrInvalidState) } @@ -429,6 +429,20 @@ func decisionInput(pending []suspension.PendingCall, d checkpoint.Decision) (Res return in, nil } +// checkDecision reports whether d can answer susp. A deferred model call +// has no tool calls, so a decision on it approves or rejects it as a whole: +// one with Calls or amended arguments is cortex.ErrInvalidDecision. Any +// other pause is checked by checkCalls. +func checkDecision(susp *suspension.Suspension, d checkpoint.Decision) error { + if isDeferredModelCall(susp) { + if len(d.Calls) > 0 || len(d.AmendedArguments) > 0 { + return fmt.Errorf("%w: a deferred model call has no tool calls to decide or amend, so approve or reject it as a whole", cortex.ErrInvalidDecision) + } + return nil + } + return checkCalls(susp.Pending, d) +} + // checkCalls reports whether d.Calls names exactly the pending calls, once // each, and nothing else. func checkCalls(pending []suspension.PendingCall, d checkpoint.Decision) error { diff --git a/engine/deferred_model_call_lease_test.go b/engine/deferred_model_call_lease_test.go new file mode 100644 index 00000000..1836fde0 --- /dev/null +++ b/engine/deferred_model_call_lease_test.go @@ -0,0 +1,97 @@ +package engine_test + +import ( + "context" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/xraph/cortex" + "github.com/xraph/cortex/checkpoint" + "github.com/xraph/cortex/engine" + "github.com/xraph/cortex/llm" + "github.com/xraph/cortex/run" +) + +// deferOnceLLM is a leaseLLM whose call number at (0-based, counting +// every call) is deferred. The deferred call reaches no response, so the +// call after it gets the response the deferred one would have had. +type deferOnceLLM struct { + *leaseLLM + at int32 + seen atomic.Int32 +} + +func (m *deferOnceLLM) Complete(ctx context.Context, req *llm.Request) (*llm.Response, error) { + if m.seen.Add(1)-1 == m.at { + return nil, cortex.DeferredModelCall{Kind: "credits", Reason: "The workspace is out of AI credits."} + } + return m.leaseLLM.Complete(ctx, req) +} + +// A deferral pauses a leased run like any other pause: the lease and the +// progress go with it. Continued, the run is claimed under a lease, and +// when the worker carrying it out stalls in the retried model call, the +// sweep reclaims it and another replica finishes it from the pause, +// without running the tool again. +func TestModelCallDeferral_reclaimableUnderLeases(t *testing.T) { + lookup := llm.Tool{Name: "lookup", Description: "test-only tool that reads something"} + var lookups atomic.Int32 + handler := func(context.Context, cortex.Invocation) (string, error) { + lookups.Add(1) + return "found", nil + } + + inner := newLeaseLLM(toolCalls(llm.ToolCall{ID: "call-1", Name: lookup.Name, Arguments: `{}`})) + // The retried call is the leaseLLM's second. + inner.hold[1] = true + model := &deferOnceLLM{leaseLLM: inner, at: 1} + f := newLeaseFixture(t, inner, engine.WithRunLease(reclaimTTL), engine.WithLLM(model), engine.WithTool(lookup, handler)) + + paused, err := f.eng.RunAgent(leaseScope(), "assistant", "work", nil) + require.NoError(t, err) + require.Equal(t, run.StatePaused, paused.State) + require.Nil(t, f.onlyRun(t).Lease, "a paused run kept its lease") + _, err = f.store.GetProgress(leaseScope(), paused.ID) + require.ErrorIs(t, err, cortex.ErrProgressNotFound) + + cps, err := f.eng.ListPendingCheckpoints(leaseScope(), &checkpoint.ListFilter{RunID: paused.ID.String()}) + require.NoError(t, err) + require.Len(t, cps, 1) + require.Equal(t, checkpoint.KindDeferredModelCall, cps[0].Metadata[checkpoint.MetadataKind]) + require.NoError(t, f.eng.DecideCheckpoint(leaseScope(), cps[0].ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"})) + + done := make(chan runOutcome, 1) + f.runs.Add(1) + go func() { + defer f.runs.Done() + r, continueErr := f.eng.ContinueRun(leaseScope(), paused.ID) + done <- runOutcome{run: r, err: continueErr} + }() + inner.waitEntered(t) + claimed := f.onlyRun(t) + require.NotNil(t, claimed.Lease, "the continued run held no lease") + require.Equal(t, f.eng.WorkerID(), claimed.Lease.Owner) + + stall(t, f, paused.ID) + bModel := &reclaimLLM{} + b := reclaimer(t, f, bModel, newReclaimHooks(), engine.WithTool(lookup, handler)) + engine.SweepOnce(b, context.Background()) + continued, err := b.ContinueRun(leaseScope(), paused.ID) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, continued.State) + require.Equal(t, 1, bModel.calls(), "the reclaiming replica did not make exactly the retried call") + require.Equal(t, "found", lastToolMessage(t, bModel.firstRequest(t)).Content, "the tool result before the deferral was lost") + require.EqualValues(t, 1, lookups.Load(), "the tool ran again") + + inner.letGo() + if out := waitOutcome(t, done); out.err == nil { + require.Equal(t, continued.Output, out.run.Output) + } else { + require.ErrorIs(t, out.err, cortex.ErrRunLeaseLost) + } + final := f.onlyRun(t) + require.Equal(t, run.StateCompleted, final.State) + require.Equal(t, continued.Output, final.Output) +} diff --git a/engine/deferred_model_call_test.go b/engine/deferred_model_call_test.go new file mode 100644 index 00000000..f4caabd1 --- /dev/null +++ b/engine/deferred_model_call_test.go @@ -0,0 +1,421 @@ +package engine + +import ( + "context" + "errors" + "fmt" + "slices" + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/xraph/cortex" + "github.com/xraph/cortex/checkpoint" + "github.com/xraph/cortex/llm" + "github.com/xraph/cortex/run" + "github.com/xraph/cortex/store" + "github.com/xraph/cortex/suspension" +) + +// deferralLLM serves its scripted responses in order and answers the +// calls named in deferAt (1-based) with that error instead. A deferred +// call consumes no response, so the call after it gets the response the +// deferred one would have had. It keeps a copy of every request's +// messages. +type deferralLLM struct { + mu sync.Mutex + responses []*llm.Response + deferAt map[int]error + // onNext makes a streamed deferral surface from the stream's first + // Next rather than from opening the stream. + onNext bool + served int + requests [][]llm.Message +} + +func (m *deferralLLM) next(req *llm.Request) (*llm.Response, error) { + m.mu.Lock() + defer m.mu.Unlock() + m.requests = append(m.requests, slices.Clone(req.Messages)) + if err, ok := m.deferAt[len(m.requests)]; ok { + return nil, err + } + resp := &llm.Response{Content: "done", FinishReason: "stop", Usage: llm.Usage{TotalTokens: 7}} + if m.served < len(m.responses) { + resp = m.responses[m.served] + } + m.served++ + return resp, nil +} + +func (m *deferralLLM) Complete(_ context.Context, req *llm.Request) (*llm.Response, error) { + return m.next(req) +} + +func (m *deferralLLM) CompleteStream(_ context.Context, req *llm.Request) (llm.Stream, error) { + resp, err := m.next(req) + if err != nil && m.onNext { + return &refusedStream{err: err}, nil + } + if err != nil { + return nil, err + } + return &outcomeStream{response: resp}, nil +} + +func (m *deferralLLM) calls() int { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.requests) +} + +func (m *deferralLLM) request(t *testing.T, i int) []llm.Message { + t.Helper() + m.mu.Lock() + defer m.mu.Unlock() + require.Greater(t, len(m.requests), i, "the model got no request %d", i) + return m.requests[i] +} + +// refusedStream fails its first Next with err, before any chunk. +type refusedStream struct{ err error } + +func (s *refusedStream) Next(context.Context) (*llm.Chunk, error) { return nil, s.err } +func (*refusedStream) Close() error { return nil } +func (*refusedStream) Usage() *llm.Usage { return nil } + +func lookupTool() llm.Tool { + return llm.Tool{Name: "lookup", Description: "test-only tool that reads something"} +} + +func creditsDeferral() error { + return cortex.DeferredModelCall{Kind: "credits", Reason: "The workspace is out of AI credits."} +} + +// deferralFixture is a continueFixture whose model calls the lookup tool +// once and then answers, deferring the calls named in deferAt. +type deferralFixture struct { + *continueFixture + model *deferralLLM + lookups atomic.Int32 +} + +func newDeferralFixture(t *testing.T, deferAt map[int]error, opts ...Option) *deferralFixture { + t.Helper() + ctx := cortex.WithScope(context.Background(), approvalScope()) + return deferralFixtureOn(ctx, t, newApprovalStore(ctx, t), deferAt, opts...) +} + +func deferralFixtureOn(ctx context.Context, t *testing.T, st store.Store, deferAt map[int]error, opts ...Option) *deferralFixture { + t.Helper() + df := &deferralFixture{model: &deferralLLM{ + responses: []*llm.Response{{ + ToolCalls: []llm.ToolCall{{ID: "call-1", Name: lookupTool().Name, Arguments: `{}`}}, + FinishReason: "tool_calls", + Usage: llm.Usage{TotalTokens: 7}, + }}, + deferAt: deferAt, + }} + base := []Option{ + WithLLM(df.model), + WithTool(lookupTool(), func(context.Context, cortex.Invocation) (string, error) { + df.lookups.Add(1) + return "found", nil + }), + } + df.continueFixture = buildContinueFixture(ctx, t, st, nil, append(base, opts...)...) + df.overrides = &RunOverrides{MaxSteps: 5} + return df +} + +// straightRun is the same run with nothing deferred: what a deferred run +// must add up to once it is continued. +func straightRun(t *testing.T) (*run.Run, *deferralFixture) { + t.Helper() + f := newDeferralFixture(t, nil) + done, err := f.e.RunAgent(as(f.ctx, alice), "assistant", "clean up", f.overrides) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, done.State) + return done, f +} + +func TestDeferredModelCall_isTheSentinel(t *testing.T) { + err := fmt.Errorf("meter: %w", cortex.DeferredModelCall{Kind: "credits", Reason: "out of credits"}) + require.ErrorIs(t, err, cortex.ErrDeferModelCall) + var deferred cortex.DeferredModelCall + require.ErrorAs(t, err, &deferred) + require.Equal(t, "credits", deferred.Kind) + require.Contains(t, err.Error(), "out of credits") +} + +func TestModelCallDeferral_suspendsBeforeTheCall(t *testing.T) { + f := newDeferralFixture(t, map[int]error{2: creditsDeferral()}) + paused, cp := f.pause(t) + require.NotNil(t, cp, "a deferral opened no checkpoint") + + require.Equal(t, 2, f.model.calls(), "the model saw a call after the deferral") + require.EqualValues(t, 1, f.lookups.Load(), "the tool did not run exactly once") + + require.Equal(t, checkpoint.KindDeferredModelCall, cp.Metadata[checkpoint.MetadataKind]) + require.Equal(t, "credits", cp.Metadata[checkpoint.MetadataDeferralKind]) + require.Equal(t, "The workspace is out of AI credits.", cp.Reason) + require.Equal(t, 1, cp.StepIndex, "the checkpoint names a step other than the deferred one") + + susp, err := f.st.GetSuspension(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, suspension.ReasonDeferredModelCall, susp.Reason) + require.Empty(t, susp.Pending) + require.Equal(t, 1, susp.Cont.StepIndex, "the run would not continue at the deferred call") + require.NotNil(t, susp.ExpiresAt, "a deferral ignored the engine's suspension TTL") + + // Only the call that ran is counted. + require.Equal(t, 7, paused.TokensUsed) + require.Equal(t, 1, paused.StepCount) + require.Equal(t, 1, paused.Usage.Steps) +} + +func TestModelCallDeferral_continueRetriesTheSameCall(t *testing.T) { + straight, sf := straightRun(t) + + f := newDeferralFixture(t, map[int]error{2: creditsDeferral()}) + paused, cp := f.pause(t) + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"})) + require.Equal(t, 2, f.model.calls(), "deciding called the model") + require.Equal(t, run.StateReady, f.runState(t, paused.ID).State) + + done, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, done.State) + require.Equal(t, "done", done.Output) + + require.Equal(t, 3, f.model.calls()) + require.Equal(t, f.model.request(t, 1), f.model.request(t, 2), "the continued call is not the deferred one") + require.Equal(t, sf.model.request(t, 1), f.model.request(t, 2), "the continued call differs from a straight run's") + require.Equal(t, sf.lookups.Load(), f.lookups.Load(), "the tool ran a different number of times than in a straight run") + require.Equal(t, straight.TokensUsed, done.TokensUsed, "the deferred call was counted") + require.Equal(t, straight.StepCount, done.StepCount) + require.Equal(t, straight.Usage.Steps, done.Usage.Steps) + + decided := f.checkpointState(t, cp.ID) + require.NotEqual(t, checkpointStatePending, decided.State) + require.NotNil(t, decided.Decision) + require.True(t, decided.Decision.Approved) +} + +func TestModelCallDeferral_firstCallOfTheRun(t *testing.T) { + f := newDeferralFixture(t, map[int]error{1: fmt.Errorf("meter: %w", creditsDeferral())}) + paused, cp := f.pause(t) + require.NotNil(t, cp) + require.Equal(t, 1, f.model.calls()) + require.Zero(t, f.lookups.Load()) + require.Zero(t, paused.TokensUsed) + require.Zero(t, paused.StepCount) + require.Zero(t, cp.StepIndex) + + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"})) + done, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, done.State) + require.Equal(t, 3, f.model.calls()) + require.Equal(t, f.model.request(t, 0), f.model.request(t, 1), "the continued call is not the deferred one") + require.EqualValues(t, 1, f.lookups.Load()) + require.Equal(t, 14, done.TokensUsed) +} + +func TestModelCallDeferral_resolveCheckpointContinuesTheRun(t *testing.T) { + f := newDeferralFixture(t, map[int]error{2: creditsDeferral()}) + paused, cp := f.pause(t) + require.NoError(t, f.e.ResolveCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"})) + done := f.runState(t, paused.ID) + require.Equal(t, run.StateCompleted, done.State) + require.Equal(t, 3, f.model.calls()) + require.EqualValues(t, 1, f.lookups.Load()) +} + +func TestModelCallDeferral_deferredAgainOnContinue(t *testing.T) { + f := newDeferralFixture(t, map[int]error{2: creditsDeferral(), 3: creditsDeferral()}) + paused, cp := f.pause(t) + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"})) + again, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, run.StatePaused, again.State) + + cps, err := f.e.ListPendingCheckpoints(f.ctx, &checkpoint.ListFilter{RunID: paused.ID.String()}) + require.NoError(t, err) + require.Len(t, cps, 1) + require.NotEqual(t, cp.ID, cps[0].ID) + require.Equal(t, checkpoint.KindDeferredModelCall, cps[0].Metadata[checkpoint.MetadataKind]) + + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cps[0].ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"})) + done, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, done.State) + require.EqualValues(t, 1, f.lookups.Load()) + require.Equal(t, 14, done.TokensUsed) +} + +func TestModelCallDeferral_rejectionStopsTheRun(t *testing.T) { + for name, opts := range map[string][]Option{ + "rejections fail runs": nil, + "rejections go back to the model": {WithRejectionsReturnToModel()}, + } { + t.Run(name, func(t *testing.T) { + f := newDeferralFixture(t, map[int]error{2: creditsDeferral()}, opts...) + paused, cp := f.pause(t) + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{DecidedBy: "bob", Reason: "Cancelled by Bob."})) + + stopped := f.runState(t, paused.ID) + require.Equal(t, run.StateCancelled, stopped.State) + require.Equal(t, run.StopCancelled, stopped.StopReason) + require.Contains(t, stopped.Error, "Cancelled by Bob.") + require.Equal(t, 2, f.model.calls()) + require.Equal(t, 1, f.ended.of(paused.ID)) + + decided := f.checkpointState(t, cp.ID) + require.NotNil(t, decided.Decision) + require.False(t, decided.Decision.Approved) + require.Equal(t, "Cancelled by Bob.", decided.Decision.Reason) + + _, err := f.st.GetSuspension(f.ctx, paused.ID) + require.ErrorIs(t, err, cortex.ErrNotSuspended) + _, err = f.e.ContinueRun(f.ctx, paused.ID) + require.ErrorIs(t, err, cortex.ErrRunAlreadyDone) + }) + } +} + +func TestModelCallDeferral_refusesAnythingButAPlainDecision(t *testing.T) { + cases := map[string]checkpoint.Decision{ + "per-call decisions": {Calls: map[string]checkpoint.CallDecision{"call-1": {Approved: true}}}, + "amended arguments": {Approved: true, AmendedArguments: []byte(`{"to":"b"}`)}, + "per-call rejection": {Calls: map[string]checkpoint.CallDecision{"call-1": {Reason: "no"}}}, + } + for name, d := range cases { + t.Run(name, func(t *testing.T) { + f := newDeferralFixture(t, map[int]error{2: creditsDeferral()}) + paused, cp := f.pause(t) + d.DecidedBy = "bob" + require.ErrorIs(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, d), cortex.ErrInvalidDecision) + require.ErrorIs(t, f.e.ResolveCheckpoint(as(f.ctx, bob), cp.ID, d), cortex.ErrInvalidDecision) + require.Equal(t, checkpointStatePending, f.checkpointState(t, cp.ID).State) + require.Equal(t, run.StatePaused, f.runState(t, paused.ID).State) + require.Equal(t, 2, f.model.calls()) + }) + } +} + +func TestModelCallDeferral_onlyADecisionContinuesIt(t *testing.T) { + f := newDeferralFixture(t, map[int]error{2: creditsDeferral()}) + paused, _ := f.pause(t) + + _, err := f.e.Resume(f.ctx, paused.ID, ResumeInput{}) + require.ErrorIs(t, err, cortex.ErrDeferModelCall) + require.ErrorIs(t, f.e.ProvideResults(f.ctx, paused.ID, ResumeInput{}), cortex.ErrDeferModelCall) + _, err = f.e.ContinueRun(f.ctx, paused.ID) + require.ErrorIs(t, err, cortex.ErrRunNotContinuable) + + require.Equal(t, run.StatePaused, f.runState(t, paused.ID).State) + require.Equal(t, 2, f.model.calls()) +} + +func TestModelCallDeferral_streamingRun(t *testing.T) { + for name, onNext := range map[string]bool{"when the stream opens": false, "on the first chunk": true} { + t.Run(name, func(t *testing.T) { + f := newDeferralFixture(t, map[int]error{2: creditsDeferral()}) + f.model.onNext = onNext + + events := make(chan StreamEvent, 64) + require.NoError(t, f.e.StreamAgent(as(f.ctx, alice), "assistant", "clean up", f.overrides, events)) + last := lastEvent(events) + require.Equal(t, EventSuspended, last.Type, "the stream ended with %v", last.Data) + require.Equal(t, string(suspension.ReasonDeferredModelCall), last.Data["reason"]) + require.Equal(t, 2, f.model.calls()) + require.EqualValues(t, 1, f.lookups.Load()) + + runs, err := f.e.ListRuns(f.ctx, &run.ListFilter{Limit: 5}) + require.NoError(t, err) + require.Len(t, runs, 1) + paused := runs[0] + require.Equal(t, run.StatePaused, paused.State) + require.Equal(t, 7, paused.TokensUsed) + cps, err := f.e.ListPendingCheckpoints(f.ctx, &checkpoint.ListFilter{RunID: paused.ID.String()}) + require.NoError(t, err) + require.Len(t, cps, 1) + require.Equal(t, checkpoint.KindDeferredModelCall, cps[0].Metadata[checkpoint.MetadataKind]) + + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cps[0].ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"})) + continued := make(chan StreamEvent, 64) + require.NoError(t, f.e.ContinueRunStream(f.ctx, paused.ID, continued)) + require.Equal(t, EventDone, lastEvent(continued).Type) + + done := f.runState(t, paused.ID) + require.Equal(t, run.StateCompleted, done.State) + require.Equal(t, 3, f.model.calls()) + require.Equal(t, f.model.request(t, 1), f.model.request(t, 2)) + require.EqualValues(t, 1, f.lookups.Load()) + require.Equal(t, 14, done.TokensUsed) + }) + } +} + +// lastEvent drains events and returns the last one. +func lastEvent(events <-chan StreamEvent) StreamEvent { + var last StreamEvent + for ev := range events { + last = ev + } + return last +} + +// A model error that is not a deferral still fails the run. +func TestModelCallDeferral_otherErrorsStillFail(t *testing.T) { + f := newDeferralFixture(t, map[int]error{2: errors.New("provider down")}) + _, err := f.e.RunAgent(as(f.ctx, alice), "assistant", "clean up", f.overrides) + require.Error(t, err) + require.NotErrorIs(t, err, cortex.ErrDeferModelCall) + runs, listErr := f.e.ListRuns(f.ctx, &run.ListFilter{Limit: 5}) + require.NoError(t, listErr) + require.Len(t, runs, 1) + require.Equal(t, run.StateFailed, runs[0].State) +} + +// A store that cannot hold ready runs resolves a deferral in place: an +// approval makes the call on the caller's goroutine, a rejection stops the +// run, and anything but a plain decision is refused. +func TestModelCallDeferral_resolvedInPlace(t *testing.T) { + inPlace := func(t *testing.T) *deferralFixture { + ctx := cortex.WithScope(context.Background(), approvalScope()) + return deferralFixtureOn(ctx, t, hiddenContinuationStore{newApprovalStore(ctx, t)}, map[int]error{2: creditsDeferral()}) + } + + t.Run("approved", func(t *testing.T) { + f := inPlace(t) + paused, cp := f.pause(t) + require.NoError(t, f.e.ResolveCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"})) + require.Equal(t, run.StateCompleted, f.runState(t, paused.ID).State) + require.Equal(t, 3, f.model.calls()) + require.Equal(t, f.model.request(t, 1), f.model.request(t, 2)) + require.EqualValues(t, 1, f.lookups.Load()) + require.True(t, f.checkpointState(t, cp.ID).Decision.Approved) + }) + t.Run("rejected", func(t *testing.T) { + f := inPlace(t) + paused, cp := f.pause(t) + require.NoError(t, f.e.ResolveCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{DecidedBy: "bob", Reason: "Cancelled by Bob."})) + stopped := f.runState(t, paused.ID) + require.Equal(t, run.StateCancelled, stopped.State) + require.Contains(t, stopped.Error, "Cancelled by Bob.") + require.Equal(t, 2, f.model.calls()) + require.False(t, f.checkpointState(t, cp.ID).Decision.Approved) + }) + t.Run("amended", func(t *testing.T) { + f := inPlace(t) + paused, cp := f.pause(t) + err := f.e.ResolveCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "bob", AmendedArguments: []byte(`{"to":"b"}`)}) + require.ErrorIs(t, err, cortex.ErrInvalidDecision) + require.Equal(t, run.StatePaused, f.runState(t, paused.ID).State) + require.Equal(t, checkpointStatePending, f.checkpointState(t, cp.ID).State) + }) +} diff --git a/engine/engine.go b/engine/engine.go index 919f6924..801b8022 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -987,10 +987,10 @@ func (e *Engine) resolveInPlace(ctx context.Context, cpID id.CheckpointID, decis if bindErr != nil { return bindErr } - if err := checkCalls(susp.Pending, decision); err != nil { + if err := checkDecision(susp, decision); err != nil { return err } - if decision.Approved || e.rejectionsToModel { + if decision.Approved || (e.rejectionsToModel && !isDeferredModelCall(susp)) { if err := e.resumeDecided(ctx, cp, susp, decision); err != nil { uncertain := errors.Is(err, cortex.ErrEffectUncertain) exhausted := errors.Is(err, cortex.ErrMaxStepsReached) @@ -1012,7 +1012,7 @@ func (e *Engine) resolveInPlace(ctx context.Context, cpID id.CheckpointID, decis } executionErr = err } - } else if err := e.failRejected(ctx, cp, susp, decision); err != nil { + } else if err := e.endRejected(ctx, cp, susp, decision); err != nil { return err } } @@ -1121,7 +1121,7 @@ func (e *Engine) failRejected(ctx context.Context, cp *checkpoint.Checkpoint, bo return fmt.Errorf("load run for rejected checkpoint: %w", err) } - susp, err := e.claimRejected(ctx, cp, bound, snapshot) + susp, err := e.claimRejected(ctx, cp, bound, snapshot, nil) if err != nil { return fmt.Errorf("claim suspension for rejected checkpoint %s: %w", cp.ID, err) } @@ -1155,13 +1155,17 @@ func (e *Engine) failRejected(ctx context.Context, cp *checkpoint.Checkpoint, bo // claimRejected claims the pause a rejection answers. With a versioned // store the claim takes bound and nothing else; a run waiting on another -// pause is cortex.ErrInvalidState. -func (e *Engine) claimRejected(ctx context.Context, cp *checkpoint.Checkpoint, bound *suspension.Suspension, snapshot *run.Run) (*suspension.Suspension, error) { +// pause is cortex.ErrInvalidState. ending, when set, goes on the leased +// claim's snapshot, so a worker that reclaims the run after a lost write +// finishes that ending instead of continuing the run. +func (e *Engine) claimRejected(ctx context.Context, cp *checkpoint.Checkpoint, bound *suspension.Suspension, snapshot *run.Run, ending *suspension.Ending) (*suspension.Suspension, error) { // A claim that takes the lease leaves a lease that lapses if the // failure below is never written, and the run is reclaimed from the // state it paused with: the person is asked again. if bound != nil { - if susp, leased, err := e.claimPauseLeased(ctx, snapshot, bound, time.Time{}, claimSnapshot(bound, nil)); leased { + first := claimSnapshot(bound, nil) + first.Ending = ending + if susp, leased, err := e.claimPauseLeased(ctx, snapshot, bound, time.Time{}, first); leased { return e.rejectedClaimResult(ctx, cp, bound, susp, err) } } @@ -1178,6 +1182,59 @@ func (e *Engine) claimRejected(ctx context.Context, cp *checkpoint.Checkpoint, b return e.rejectedClaimResult(ctx, cp, bound, susp, err) } +// endRejected ends the run behind a rejected checkpoint when the rejection +// leaves nothing to continue: a deferred model call is stopped, any other +// pause is failed. +func (e *Engine) endRejected(ctx context.Context, cp *checkpoint.Checkpoint, bound *suspension.Suspension, decision checkpoint.Decision) error { + if isDeferredModelCall(bound) { + return e.stopDeferred(ctx, cp, bound, decision) + } + return e.failRejected(ctx, cp, bound, decision) +} + +// stopDeferred ends a run whose deferred model call was rejected, which is +// how a host stops a run that waits on one: the run is cancelled, with the +// decider's reason as its error, and the pause is dropped. The deferred +// call is never made. +// +// It claims the pause the way failRejected does, so a decision racing +// another one, or a cancel, is refused rather than written over it. A +// leased claim records the cancel on its snapshot, so a worker that +// reclaims the run after a lost write finishes the cancel rather than +// making the call. The checkpoint is left for the caller to record the +// decision on. +func (e *Engine) stopDeferred(ctx context.Context, cp *checkpoint.Checkpoint, bound *suspension.Suspension, decision checkpoint.Decision) error { + snapshot, err := e.store.GetRun(ctx, cp.RunID) + if err != nil { + return fmt.Errorf("load run for rejected checkpoint: %w", err) + } + susp, err := e.claimRejected(ctx, cp, bound, snapshot, &suspension.Ending{Kind: suspension.EndingCancel}) + if err != nil { + return fmt.Errorf("claim suspension for rejected checkpoint %s: %w", cp.ID, err) + } + ctx = cortex.WithScope(ctx, susp.Scope) + r := e.claimedRun(ctx, cp.RunID, snapshot, susp.ClaimedRun) + + reason := decision.Reason + if reason == "" { + reason = "no reason given" + } + r.State = run.StateCancelled + r.StopReason = run.StopCancelled + r.Error = "stopped while its model call was deferred: " + reason + completedAt := time.Now().UTC() + r.CompletedAt = &completedAt + if err := e.persistRun(context.WithoutCancel(ctx), r); err != nil { + // The suspension and the checkpoint stay, as in failRejected. + return fmt.Errorf("stop run %s: %w", cp.RunID, err) + } + e.emitTerminalOutcome(context.WithoutCancel(ctx), r) + if err := e.store.DeleteSuspension(ctx, cp.RunID); err != nil && !errors.Is(err, cortex.ErrNotSuspended) { + e.logError("delete suspension of a stopped run", err, log.String("run_id", cp.RunID.String())) + } + return nil +} + // rejectedClaimResult maps a pinned rejection claim's refusal: a run // waiting on another pause is cortex.ErrInvalidState. func (e *Engine) rejectedClaimResult(ctx context.Context, cp *checkpoint.Checkpoint, bound, susp *suspension.Suspension, err error) (*suspension.Suspension, error) { diff --git a/engine/react.go b/engine/react.go index 1e95e7d2..cf38600b 100644 --- a/engine/react.go +++ b/engine/react.go @@ -244,6 +244,18 @@ func (e *Engine) continueReAct(ctx context.Context, ag *agent.Config, cfg resolv fresh := progressCont(r, st, cfg) return r, e.leaveRun(ctx, r, &fresh) } + if errors.Is(err, cortex.ErrDeferModelCall) { + // The client will make this call later, not now: the run + // pauses before it, and a decision continues it with the + // same call. Nothing is charged for a call that did not run. + if _, suspendErr := e.suspendDeferred(ctx, r, st, cfg, err); suspendErr != nil { + e.failRun(ctx, r, ag.ID, suspendErr, startedAt) + stepSpan.finish(stepError, suspendErr) + return nil, fmt.Errorf("suspend run: %w", suspendErr) + } + stepSpan.finish(stepSuspended, nil) + return r, nil + } if errors.Is(err, llm.ErrContentBlocked) { reported := 0 if resp != nil { @@ -606,6 +618,11 @@ func (e *Engine) continueStreamReAct(ctx context.Context, ag *agent.Config, cfg sendLeftEvent(ctx, events, r, e.leaveRun(ctx, r, &fresh)) return } + if errors.Is(err, cortex.ErrDeferModelCall) { + // Same pause as the synchronous loop. + e.suspendDeferredStream(ctx, ag, r, st, cfg, err, stepSpan, startedAt, events) + return + } if errors.Is(err, llm.ErrContentBlocked) { terminalErr := e.stopContentBlocked(ctx, r, st, cfg, 0) stepSpan.finish(stepStopped, nil) @@ -685,6 +702,16 @@ func (e *Engine) continueStreamReAct(ctx context.Context, ag *agent.Config, cfg sendLeftEvent(ctx, events, r, e.leaveRun(ctx, r, &fresh)) return } + if errors.Is(err, cortex.ErrDeferModelCall) && totalBytes == 0 { + // A client may defer the call from the stream's first + // read rather than from opening it. Nothing of the + // response was read, so the run pauses before the call as + // it does above. Once a chunk has arrived the call has + // run, and a deferral fails the run like any error. + _ = stream.Close() + e.suspendDeferredStream(ctx, ag, r, st, cfg, err, stepSpan, startedAt, events) + return + } if errors.Is(err, llm.ErrContentBlocked) { // Nothing from a blocked stream runs: the tool-call deltas // already read are dropped with it. @@ -994,6 +1021,26 @@ func (e *Engine) continueStreamReAct(ctx context.Context, ag *agent.Config, cfg sendEvent(ctx, events, outcomeEvent(r, completedAt.Sub(startedAt))) } +// suspendDeferredStream is the streaming loop's pause for a deferred +// model call: suspendDeferred, and the suspended event that tells the +// consumer the run paused rather than ended. A pause that cannot be +// written fails the run. +func (e *Engine) suspendDeferredStream(ctx context.Context, ag *agent.Config, r *run.Run, st *reactState, cfg resolvedConfig, cause error, stepSpan *stepTrace, startedAt time.Time, events chan<- StreamEvent) { + reason, err := e.suspendDeferred(ctx, r, st, cfg, cause) + if err != nil { + e.failRun(ctx, r, ag.ID, err, startedAt) + stepSpan.finish(stepError, err) + sendEvent(ctx, events, runErrorEvent(r, err)) + return + } + stepSpan.finish(stepSuspended, nil) + sendEvent(ctx, events, StreamEvent{Type: EventSuspended, Data: map[string]any{ + "run_id": r.ID.String(), + "reason": string(reason), + "pending": publicPendingCalls(r, nil), + }}) +} + // sendLeftEvent tells a stream's consumer how a pass that left its run // ended: the run's own outcome when it turned out to have ended already // (err is nil), otherwise the hand-off or the lost lease. diff --git a/engine/resume.go b/engine/resume.go index 9496257b..f7886d0b 100644 --- a/engine/resume.go +++ b/engine/resume.go @@ -740,6 +740,11 @@ func (e *Engine) failResume(ctx context.Context, r *run.Run, agentID id.AgentID, // exception to completed or failed: the call was refused by a person, so // it emits ToolDenied, the event an authorizer's refusal emits. func (e *Engine) recordResumedCalls(ctx context.Context, ag *agent.Config, r *run.Run, susp *suspension.Suspension, in ResumeInput, st *reactState, cfg resolvedConfig) error { + if len(susp.Pending) == 0 { + // A deferred model call: nothing to record, the loop makes the + // call next. + return nil + } ctx = withOperationalRuntime(ctx, r, cfg, st) byID := make(map[string]ToolResult, len(in.ToolResults)) for _, res := range in.ToolResults { @@ -1015,6 +1020,15 @@ func checkResumeAuthority(reason suspension.SuspendReason, source resumeSource) } return fmt.Errorf("%w: this run is waiting on another agent's reply, which only the messaging bus can deliver", ErrNotAgentReplyResumable) + case suspension.ReasonDeferredModelCall: + // Nothing outside the engine has a result to give a deferred + // model call. The run continues when its checkpoint is decided, + // and then it makes the call itself. + if source == resumeSourceApproval { + return nil + } + return fmt.Errorf("%w: this run is waiting to make a model call it deferred, so decide its checkpoint rather than resuming the run", cortex.ErrDeferModelCall) + default: return nil } diff --git a/engine/suspend.go b/engine/suspend.go index c0756f85..446b9197 100644 --- a/engine/suspend.go +++ b/engine/suspend.go @@ -86,6 +86,11 @@ type pendingCalls struct { // run is the run this step belongs to, and the only one whose asks // it collects. run id.AgentRunID + + // deferred is set, and calls empty, when the step's model call was + // deferred by the model client rather than answered: the run waits + // before that call, with nothing pending. + deferred *cortex.DeferredModelCall } func (p *pendingCalls) add(tc llm.ToolCall, reason suspension.SuspendReason, ttl time.Duration, note string) { @@ -134,6 +139,9 @@ func stepPendingCalls(ctx context.Context, runID id.AgentRunID) *pendingCalls { // asked the engine to run, three steps away from anything that explains // it. func (p *pendingCalls) reason() (suspension.SuspendReason, error) { + if p.deferred != nil { + return suspension.ReasonDeferredModelCall, nil + } if len(p.reasons) == 0 { return "", errors.New("suspend was asked for with no pending calls") } @@ -245,7 +253,7 @@ func (e *Engine) suspendRun(ctx context.Context, r *run.Run, pend pendingCalls, // decision nobody can see: ListPending never shows it, so nothing // resolves it. Written here, a failed checkpoint write takes the // suspension down with it and the run never reaches paused at all. - cp, err := e.createCheckpoint(ctx, r, s, reason) + cp, err := e.createCheckpoint(ctx, r, s, reason, pend.deferred) if err != nil { if delErr := e.store.DeleteSuspension(ctx, r.ID); delErr != nil { e.logError("delete suspension after a failed checkpoint write", delErr, log.String("run_id", r.ID.String())) @@ -378,6 +386,12 @@ func (e *Engine) suspensionDeadline(pend pendingCalls, reason suspension.Suspend } var earliest *time.Time + if pend.deferred != nil && e.suspensionTTL > 0 { + // A deferred model call has no pending call to bring a TTL of its + // own, so it waits as long as the engine's suspension TTL allows. + due := now.Add(e.suspensionTTL).UTC() + earliest = &due + } for i, call := range pend.calls { var due time.Time var ttl time.Duration @@ -418,7 +432,10 @@ func (e *Engine) suspensionDeadline(pend pendingCalls, reason suspension.Suspend // answered by the host that registered the tool, not by a person, and a // checkpoint queue full of rows no human is meant to decide is a queue // people stop reading. -func (e *Engine) createCheckpoint(ctx context.Context, r *run.Run, s *suspension.Suspension, reason suspension.SuspendReason) (*checkpoint.Checkpoint, error) { +func (e *Engine) createCheckpoint(ctx context.Context, r *run.Run, s *suspension.Suspension, reason suspension.SuspendReason, deferred *cortex.DeferredModelCall) (*checkpoint.Checkpoint, error) { + if reason == suspension.ReasonDeferredModelCall && deferred != nil { + return e.createDeferralCheckpoint(ctx, r, s, *deferred) + } if reason != suspension.ReasonApproval { return nil, nil } @@ -468,3 +485,63 @@ func (e *Engine) createCheckpoint(ctx context.Context, r *run.Run, s *suspension // the pause is real. return cp, nil } + +// deferredCallReason is the checkpoint reason of a deferred model call +// whose client gave none. +const deferredCallReason = "the model call was deferred" + +// createDeferralCheckpoint opens the checkpoint of a deferred model call: +// what the host decides on to continue the run (approve, and the same +// call is made) or to stop it (reject). The checkpoint carries the +// client's reason, and its metadata names the kind of checkpoint and the +// kind of wait, so a host can tell it from a tool approval and knows what +// it waits on. +func (e *Engine) createDeferralCheckpoint(ctx context.Context, r *run.Run, s *suspension.Suspension, deferred cortex.DeferredModelCall) (*checkpoint.Checkpoint, error) { + reason := deferred.Reason + if reason == "" { + reason = deferredCallReason + } + cp := &checkpoint.Checkpoint{ + Entity: cortex.NewEntity(), + ID: id.NewCheckpointID(), + RunID: r.ID, + AgentID: r.AgentID, + Scope: r.Scope, + Reason: reason, + // The deferred call is the next step's, the one the run continues + // with. + StepIndex: s.Cont.StepIndex, + State: checkpointStatePending, + Metadata: map[string]any{ + checkpointSuspensionKey: s.ID.String(), + checkpoint.MetadataKind: checkpoint.KindDeferredModelCall, + checkpoint.MetadataDeferralKind: deferred.Kind, + }, + } + if err := e.store.CreateCheckpoint(ctx, cp); err != nil { + return nil, err + } + return cp, nil +} + +// suspendDeferred pauses a run whose model client deferred the step's +// model call (cortex.ErrDeferModelCall). The continuation is the state +// from before that call, so nothing the run already did is repeated when +// it continues, and the deferred call itself is not counted: it has not +// run yet. A decision on the checkpoint continues the run, which makes the +// same call again. +func (e *Engine) suspendDeferred(ctx context.Context, r *run.Run, st *reactState, cfg resolvedConfig, cause error) (suspension.SuspendReason, error) { + var deferred cortex.DeferredModelCall + if !errors.As(cause, &deferred) { + // A client that wrapped the bare sentinel said nothing more. + deferred = cortex.DeferredModelCall{} + } + st.provenance = r.Provenance + return e.suspend(ctx, r, pendingCalls{run: r.ID, deferred: &deferred}, st.continuation(cfg)) +} + +// isDeferredModelCall reports whether a suspension waits on a deferred +// model call rather than on tool calls. +func isDeferredModelCall(susp *suspension.Suspension) bool { + return susp != nil && susp.Reason == suspension.ReasonDeferredModelCall +} diff --git a/engine/sweeper.go b/engine/sweeper.go index ea818227..0382425e 100644 --- a/engine/sweeper.go +++ b/engine/sweeper.go @@ -549,5 +549,8 @@ func expiredDetail(susp *suspension.Suspension) string { if susp.ExpiresAt != nil { deadline = susp.ExpiresAt.Format(time.RFC3339) } + if isDeferredModelCall(susp) { + return "nothing continued the deferred model call by " + deadline + } return fmt.Sprintf("nothing answered the %s pending call(s) by %s", susp.Reason, deadline) } diff --git a/engine/telemetry.go b/engine/telemetry.go index ca41f71d..23935140 100644 --- a/engine/telemetry.go +++ b/engine/telemetry.go @@ -275,7 +275,7 @@ func purposeLabel(p run.ExecutionPurpose) string { func suspendReasonLabel(r suspension.SuspendReason) string { switch r { - case suspension.ReasonApproval, suspension.ReasonExternalTool, suspension.ReasonAgentReply: + case suspension.ReasonApproval, suspension.ReasonExternalTool, suspension.ReasonAgentReply, suspension.ReasonDeferredModelCall: return string(r) default: return labelUnknown diff --git a/errors.go b/errors.go index 9aaf443d..08b4d6ab 100644 --- a/errors.go +++ b/errors.go @@ -237,3 +237,38 @@ var ( // returned when a caller's own results carry what only a decision may set // (edited arguments or the mark of a reviewer's denial). var ErrInvalidDecision = errors.New("cortex: invalid checkpoint decision") + +// ErrDeferModelCall is what a model client returns, wrapped or as +// DeferredModelCall, when it will not make a call right now but may later: +// the workspace is out of AI credits, say. The run pauses before that call +// with a checkpoint, nothing the run already did is lost, and an approved +// decision on the checkpoint continues the run by making the same call. +// Any other model error still fails the run. +var ErrDeferModelCall = errors.New("cortex: model call deferred") + +// DeferredModelCall is ErrDeferModelCall with something to say: why the +// call waits (Reason, which becomes the checkpoint's reason) and what kind +// of wait it is (Kind, for example "credits", which the host reads back +// from the checkpoint's metadata). Return it from llm.Client, bare or +// wrapped; errors.Is matches it against ErrDeferModelCall. +type DeferredModelCall struct { + // Reason is shown on the checkpoint. + Reason string + // Kind names what the call waits on. The engine does not interpret + // it. + Kind string +} + +// Error says the model call was deferred, and why. +func (d DeferredModelCall) Error() string { + if d.Reason == "" { + return ErrDeferModelCall.Error() + } + return ErrDeferModelCall.Error() + ": " + d.Reason +} + +// Is reports a match for ErrDeferModelCall, so code that only knows the +// sentinel still recognizes a deferral. +func (d DeferredModelCall) Is(target error) bool { + return target == ErrDeferModelCall +} diff --git a/store/storetest/conformance.go b/store/storetest/conformance.go index 81207b73..29d3a66f 100644 --- a/store/storetest/conformance.go +++ b/store/storetest/conformance.go @@ -2054,6 +2054,38 @@ func testSameIdentifierCrossScope(t *testing.T, newStore func(t *testing.T) stor } }) + // A deferred model call pauses a run with no pending calls. The row + // reads back with its reason and an empty pending list, and a claim + // takes it like any other pause. + t.Run("SuspensionWithoutPendingCalls", func(t *testing.T) { + s := newStore(t) + ctx := ctxWithScope("ws_deferred_model_call") + r := mustPauseRun(t, s, ctx) + susp := newSuspension(r.ID) + susp.Reason = suspension.ReasonDeferredModelCall + susp.Pending = nil + if err := s.CreateSuspension(ctx, susp); err != nil { + t.Fatalf("create suspension: %v", err) + } + got, err := s.GetSuspension(ctx, r.ID) + if err != nil { + t.Fatalf("get suspension: %v", err) + } + if got.Reason != suspension.ReasonDeferredModelCall || len(got.Pending) != 0 { + t.Fatalf("suspension read back as (%q, %+v), want (%q, no pending calls)", got.Reason, got.Pending, suspension.ReasonDeferredModelCall) + } + if got.Cont.StepIndex != susp.Cont.StepIndex || len(got.Cont.Messages) != 1 { + t.Fatalf("continuation read back as %+v, want %+v", got.Cont, susp.Cont) + } + claimed, err := s.ClaimSuspension(ctx, r.ID) + if err != nil { + t.Fatalf("claim suspension: %v", err) + } + if claimed.ID != susp.ID || len(claimed.Pending) != 0 { + t.Fatalf("claimed %+v, want suspension %s with no pending calls", claimed, susp.ID) + } + }) + // A decision's amended arguments are stored with it and read back // the same JSON values (jsonb re-spaces them), whichever backend holds the checkpoint. t.Run("CheckpointDecisionAmendedArguments", func(t *testing.T) { diff --git a/suspension/suspension.go b/suspension/suspension.go index e4e3b393..5c2125d1 100644 --- a/suspension/suspension.go +++ b/suspension/suspension.go @@ -38,6 +38,12 @@ const ( // sent. The messaging bus resumes these, off its own correlation // ledger; nobody else may. ReasonAgentReply SuspendReason = "agent_reply" + // ReasonDeferredModelCall means the model client deferred the run's + // next model call (cortex.DeferredModelCall). The suspension has no + // pending calls, its continuation is the state from before that call, + // and only a decision on its checkpoint continues it, by making the + // same call. + ReasonDeferredModelCall SuspendReason = "deferred_model_call" ) // PendingCall is one tool call awaiting a result from outside the engine. @@ -72,7 +78,7 @@ type Notice struct { // ExpiresAt is the suspension's deadline, nil when it has none. ExpiresAt *time.Time // CheckpointID is the checkpoint a person decides. It is zero unless - // Reason is ReasonApproval. + // Reason is ReasonApproval or ReasonDeferredModelCall. CheckpointID id.CheckpointID Scope cortex.Scope } From f83df9fa5ff25d6528ed40403c348454e5ea645c Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Fri, 13 Nov 2026 18:43:26 -0600 Subject: [PATCH 6/9] fix(engine): treat an empty tool list as none when continuing a run --- engine/deferred_model_call_test.go | 18 ++++++++++++++++++ engine/prepared_resume.go | 9 +++++++++ 2 files changed, 27 insertions(+) diff --git a/engine/deferred_model_call_test.go b/engine/deferred_model_call_test.go index f4caabd1..db1d92fe 100644 --- a/engine/deferred_model_call_test.go +++ b/engine/deferred_model_call_test.go @@ -419,3 +419,21 @@ func TestModelCallDeferral_resolvedInPlace(t *testing.T) { require.Equal(t, checkpointStatePending, f.checkpointState(t, cp.ID).State) }) } + +// An empty allowlist and no allowlist freeze to the same tools: a run +// started with Tools set to an empty slice can be continued, although the +// continuation stores no tools at all. +func TestModelCallDeferral_continuesARunWithAnEmptyToolList(t *testing.T) { + f := newDeferralFixture(t, map[int]error{1: creditsDeferral()}) + f.model.responses = nil + f.overrides = &RunOverrides{MaxSteps: 5, Tools: []string{}, ToolsRestricted: true} + paused, cp := f.pause(t) + require.NotNil(t, cp) + + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"})) + done, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, done.State) + require.Equal(t, "done", done.Output) + require.Zero(t, f.lookups.Load()) +} diff --git a/engine/prepared_resume.go b/engine/prepared_resume.go index 6188ab1b..156274f1 100644 --- a/engine/prepared_resume.go +++ b/engine/prepared_resume.go @@ -59,6 +59,15 @@ func validateFrozenContinuation(cont suspension.Continuation, snap *preparedSnap return cortex.ErrInvalidContinuation } cfg.Budget.Deadline = frozen.Budget.Deadline + // The continuation drops an empty tool list (omitempty) while the + // snapshot keeps it. Both mean no listed tools; ToolsRestricted says + // whether that is an explicit allowlist, and it is compared as is. + if len(cfg.Tools) == 0 { + cfg.Tools = nil + } + if len(frozen.Tools) == 0 { + frozen.Tools = nil + } if !reflect.DeepEqual(cfg, frozen) || cont.SystemPrompt != prompt.Assemble(snap.Sections) || r.AgentID != snap.Agent.ID || r.Scope.Canonical() != snap.Scope.Canonical() { return fmt.Errorf("cortex: continuation changed frozen inputs: %w", cortex.ErrInvalidContinuation) } From 5c377cd702d6db9b1ac57b1a491fcab7732d3d87 Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Fri, 13 Nov 2026 19:21:11 -0600 Subject: [PATCH 7/9] fix(lint): name deferral errors and stop shadowing err --- checkpoint/checkpoint.go | 4 ++-- engine/continue.go | 8 ++++---- engine/deferred_model_call_lease_test.go | 2 +- engine/deferred_model_call_test.go | 11 ++++++----- engine/resume.go | 2 +- engine/suspend.go | 10 +++++----- errors.go | 10 +++++----- store/storetest/conformance.go | 14 +++++++------- suspension/suspension.go | 2 +- 9 files changed, 32 insertions(+), 31 deletions(-) diff --git a/checkpoint/checkpoint.go b/checkpoint/checkpoint.go index 59e4f276..0f16a71c 100644 --- a/checkpoint/checkpoint.go +++ b/checkpoint/checkpoint.go @@ -15,7 +15,7 @@ import ( const MetadataKind = "kind" // KindDeferredModelCall is the MetadataKind of a checkpoint opened because -// the model client deferred a call (cortex.DeferredModelCall). Such a +// the model client deferred a call (cortex.DeferredModelCallError). Such a // checkpoint has no pending tool calls. An approval continues the run with // the same model call; a rejection stops the run with the decision's // reason. A decision that carries Calls or AmendedArguments is refused @@ -23,7 +23,7 @@ const MetadataKind = "kind" const KindDeferredModelCall = "deferred_model_call" // MetadataDeferralKind is the metadata key that carries -// cortex.DeferredModelCall.Kind on a deferred model call's checkpoint. +// cortex.DeferredModelCallError.Kind on a deferred model call's checkpoint. const MetadataDeferralKind = "deferral_kind" // Decision represents the resolution of a checkpoint. diff --git a/engine/continue.go b/engine/continue.go index 9bd5d0be..bfcca620 100644 --- a/engine/continue.go +++ b/engine/continue.go @@ -80,8 +80,8 @@ func (e *Engine) decideCheckpoint(ctx context.Context, cpID id.CheckpointID, d c if err != nil { return nil, err } - if err := checkDecision(susp, d); err != nil { - return nil, err + if decideErr := checkDecision(susp, d); decideErr != nil { + return nil, decideErr } if !d.Approved && (!e.rejectionsToModel || isDeferredModelCall(susp)) { if rejectErr := e.endRejected(ctx, cp, susp, d); rejectErr != nil { @@ -944,7 +944,7 @@ func settleCalls(d checkpoint.Decision) checkpoint.Decision { // from the results its run was made ready with, for a record that was lost. // A single call is recorded the top-level way, several are recorded in // Calls, where a rejection keeps the reason the model was given. -func decisionFromResults(ready *run.Readiness) (json.RawMessage, map[string]checkpoint.CallDecision) { +func decisionFromResults(ready *run.Readiness) (amended json.RawMessage, calls map[string]checkpoint.CallDecision) { var results []ToolResult if json.Unmarshal(ready.Results, &results) != nil { return nil, nil @@ -966,7 +966,7 @@ func decisionFromResults(ready *run.Readiness) (json.RawMessage, map[string]chec by = "a reviewer" } prefix := "rejected by " + by + ": " - calls := make(map[string]checkpoint.CallDecision, len(results)) + calls = make(map[string]checkpoint.CallDecision, len(results)) for _, res := range results { if res.Execute { calls[res.ToolCallID] = checkpoint.CallDecision{Approved: true, AmendedArguments: res.Arguments} diff --git a/engine/deferred_model_call_lease_test.go b/engine/deferred_model_call_lease_test.go index 1836fde0..4addfd26 100644 --- a/engine/deferred_model_call_lease_test.go +++ b/engine/deferred_model_call_lease_test.go @@ -25,7 +25,7 @@ type deferOnceLLM struct { func (m *deferOnceLLM) Complete(ctx context.Context, req *llm.Request) (*llm.Response, error) { if m.seen.Add(1)-1 == m.at { - return nil, cortex.DeferredModelCall{Kind: "credits", Reason: "The workspace is out of AI credits."} + return nil, cortex.DeferredModelCallError{Kind: "credits", Reason: "The workspace is out of AI credits."} } return m.leaseLLM.Complete(ctx, req) } diff --git a/engine/deferred_model_call_test.go b/engine/deferred_model_call_test.go index db1d92fe..bf1eb1b0 100644 --- a/engine/deferred_model_call_test.go +++ b/engine/deferred_model_call_test.go @@ -91,7 +91,7 @@ func lookupTool() llm.Tool { } func creditsDeferral() error { - return cortex.DeferredModelCall{Kind: "credits", Reason: "The workspace is out of AI credits."} + return cortex.DeferredModelCallError{Kind: "credits", Reason: "The workspace is out of AI credits."} } // deferralFixture is a continueFixture whose model calls the lookup tool @@ -118,13 +118,14 @@ func deferralFixtureOn(ctx context.Context, t *testing.T, st store.Store, deferA }}, deferAt: deferAt, }} - base := []Option{ + base := make([]Option, 0, 2+len(opts)) + base = append(base, WithLLM(df.model), WithTool(lookupTool(), func(context.Context, cortex.Invocation) (string, error) { df.lookups.Add(1) return "found", nil }), - } + ) df.continueFixture = buildContinueFixture(ctx, t, st, nil, append(base, opts...)...) df.overrides = &RunOverrides{MaxSteps: 5} return df @@ -142,9 +143,9 @@ func straightRun(t *testing.T) (*run.Run, *deferralFixture) { } func TestDeferredModelCall_isTheSentinel(t *testing.T) { - err := fmt.Errorf("meter: %w", cortex.DeferredModelCall{Kind: "credits", Reason: "out of credits"}) + err := fmt.Errorf("meter: %w", cortex.DeferredModelCallError{Kind: "credits", Reason: "out of credits"}) require.ErrorIs(t, err, cortex.ErrDeferModelCall) - var deferred cortex.DeferredModelCall + var deferred cortex.DeferredModelCallError require.ErrorAs(t, err, &deferred) require.Equal(t, "credits", deferred.Kind) require.Contains(t, err.Error(), "out of credits") diff --git a/engine/resume.go b/engine/resume.go index f7886d0b..19b3eaca 100644 --- a/engine/resume.go +++ b/engine/resume.go @@ -189,7 +189,7 @@ func (e *Engine) resumeRun(ctx context.Context, runID id.AgentRunID, in ResumeIn // when execution completes, same as StreamAgent's, and the run's admission // slot is held until then. func (e *Engine) ResumeStream(ctx context.Context, runID id.AgentRunID, in ResumeInput, events chan<- StreamEvent) (err error) { - if err := refuseDecisionFields(in.ToolResults); err != nil { + if err = refuseDecisionFields(in.ToolResults); err != nil { close(events) return err } diff --git a/engine/suspend.go b/engine/suspend.go index 446b9197..7e9a194c 100644 --- a/engine/suspend.go +++ b/engine/suspend.go @@ -90,7 +90,7 @@ type pendingCalls struct { // deferred is set, and calls empty, when the step's model call was // deferred by the model client rather than answered: the run waits // before that call, with nothing pending. - deferred *cortex.DeferredModelCall + deferred *cortex.DeferredModelCallError } func (p *pendingCalls) add(tc llm.ToolCall, reason suspension.SuspendReason, ttl time.Duration, note string) { @@ -432,7 +432,7 @@ func (e *Engine) suspensionDeadline(pend pendingCalls, reason suspension.Suspend // answered by the host that registered the tool, not by a person, and a // checkpoint queue full of rows no human is meant to decide is a queue // people stop reading. -func (e *Engine) createCheckpoint(ctx context.Context, r *run.Run, s *suspension.Suspension, reason suspension.SuspendReason, deferred *cortex.DeferredModelCall) (*checkpoint.Checkpoint, error) { +func (e *Engine) createCheckpoint(ctx context.Context, r *run.Run, s *suspension.Suspension, reason suspension.SuspendReason, deferred *cortex.DeferredModelCallError) (*checkpoint.Checkpoint, error) { if reason == suspension.ReasonDeferredModelCall && deferred != nil { return e.createDeferralCheckpoint(ctx, r, s, *deferred) } @@ -496,7 +496,7 @@ const deferredCallReason = "the model call was deferred" // client's reason, and its metadata names the kind of checkpoint and the // kind of wait, so a host can tell it from a tool approval and knows what // it waits on. -func (e *Engine) createDeferralCheckpoint(ctx context.Context, r *run.Run, s *suspension.Suspension, deferred cortex.DeferredModelCall) (*checkpoint.Checkpoint, error) { +func (e *Engine) createDeferralCheckpoint(ctx context.Context, r *run.Run, s *suspension.Suspension, deferred cortex.DeferredModelCallError) (*checkpoint.Checkpoint, error) { reason := deferred.Reason if reason == "" { reason = deferredCallReason @@ -531,10 +531,10 @@ func (e *Engine) createDeferralCheckpoint(ctx context.Context, r *run.Run, s *su // run yet. A decision on the checkpoint continues the run, which makes the // same call again. func (e *Engine) suspendDeferred(ctx context.Context, r *run.Run, st *reactState, cfg resolvedConfig, cause error) (suspension.SuspendReason, error) { - var deferred cortex.DeferredModelCall + var deferred cortex.DeferredModelCallError if !errors.As(cause, &deferred) { // A client that wrapped the bare sentinel said nothing more. - deferred = cortex.DeferredModelCall{} + deferred = cortex.DeferredModelCallError{} } st.provenance = r.Provenance return e.suspend(ctx, r, pendingCalls{run: r.ID, deferred: &deferred}, st.continuation(cfg)) diff --git a/errors.go b/errors.go index 08b4d6ab..248ddd8f 100644 --- a/errors.go +++ b/errors.go @@ -239,19 +239,19 @@ var ( var ErrInvalidDecision = errors.New("cortex: invalid checkpoint decision") // ErrDeferModelCall is what a model client returns, wrapped or as -// DeferredModelCall, when it will not make a call right now but may later: +// DeferredModelCallError, when it will not make a call right now but may later: // the workspace is out of AI credits, say. The run pauses before that call // with a checkpoint, nothing the run already did is lost, and an approved // decision on the checkpoint continues the run by making the same call. // Any other model error still fails the run. var ErrDeferModelCall = errors.New("cortex: model call deferred") -// DeferredModelCall is ErrDeferModelCall with something to say: why the +// DeferredModelCallError is ErrDeferModelCall with something to say: why the // call waits (Reason, which becomes the checkpoint's reason) and what kind // of wait it is (Kind, for example "credits", which the host reads back // from the checkpoint's metadata). Return it from llm.Client, bare or // wrapped; errors.Is matches it against ErrDeferModelCall. -type DeferredModelCall struct { +type DeferredModelCallError struct { // Reason is shown on the checkpoint. Reason string // Kind names what the call waits on. The engine does not interpret @@ -260,7 +260,7 @@ type DeferredModelCall struct { } // Error says the model call was deferred, and why. -func (d DeferredModelCall) Error() string { +func (d DeferredModelCallError) Error() string { if d.Reason == "" { return ErrDeferModelCall.Error() } @@ -269,6 +269,6 @@ func (d DeferredModelCall) Error() string { // Is reports a match for ErrDeferModelCall, so code that only knows the // sentinel still recognizes a deferral. -func (d DeferredModelCall) Is(target error) bool { +func (d DeferredModelCallError) Is(target error) bool { return target == ErrDeferModelCall } diff --git a/store/storetest/conformance.go b/store/storetest/conformance.go index 29d3a66f..73b3c991 100644 --- a/store/storetest/conformance.go +++ b/store/storetest/conformance.go @@ -2105,10 +2105,10 @@ func testSameIdentifierCrossScope(t *testing.T, newStore func(t *testing.T) stor t.Fatal("reloaded checkpoint has no decision") } var want, have map[string]any - if err := json.Unmarshal(amended, &want); err != nil { + if err = json.Unmarshal(amended, &want); err != nil { t.Fatalf("decode want: %v", err) } - if err := json.Unmarshal(got.Decision.AmendedArguments, &have); err != nil { + if err = json.Unmarshal(got.Decision.AmendedArguments, &have); err != nil { t.Fatalf("decode amended arguments %q: %v", got.Decision.AmendedArguments, err) } if !reflect.DeepEqual(want, have) { @@ -2121,7 +2121,7 @@ func testSameIdentifierCrossScope(t *testing.T, newStore func(t *testing.T) stor "call-1": {Approved: true, AmendedArguments: json.RawMessage(`{"to":"b"}`)}, "call-2": {Reason: "wrong recipient"}, } - if err := s.Resolve(ctx, cp3.ID, checkpoint.Decision{Approved: true, DecidedBy: "reviewer", Calls: perCall}); err != nil { + if err = s.Resolve(ctx, cp3.ID, checkpoint.Decision{Approved: true, DecidedBy: "reviewer", Calls: perCall}); err != nil { t.Fatalf("Resolve with calls: %v", err) } got3, err := s.GetCheckpoint(ctx, cp3.ID) @@ -2134,8 +2134,8 @@ func testSameIdentifierCrossScope(t *testing.T, newStore func(t *testing.T) stor c1, c2 := got3.Decision.Calls["call-1"], got3.Decision.Calls["call-2"] // jsonb re-spaces JSON, so compare the objects, not the bytes. var c1Args map[string]any - if err := json.Unmarshal(c1.AmendedArguments, &c1Args); err != nil || !reflect.DeepEqual(c1Args, map[string]any{"to": "b"}) { - t.Fatalf("call-1 amended arguments = %s (%v), want {\"to\":\"b\"}", c1.AmendedArguments, err) + if decodeErr := json.Unmarshal(c1.AmendedArguments, &c1Args); decodeErr != nil || !reflect.DeepEqual(c1Args, map[string]any{"to": "b"}) { + t.Fatalf("call-1 amended arguments = %s (%v), want {\"to\":\"b\"}", c1.AmendedArguments, decodeErr) } if !c1.Approved || c2.Approved || c2.Reason != "wrong recipient" || len(c2.AmendedArguments) != 0 { t.Fatalf("calls read back as %+v, want %+v", got3.Decision.Calls, perCall) @@ -2143,8 +2143,8 @@ func testSameIdentifierCrossScope(t *testing.T, newStore func(t *testing.T) stor // A decision without them reads back without them. cp2 := mustCreateCheckpoint(t, s, ctx, r.ID, r.AgentID) - if err := s.Resolve(ctx, cp2.ID, checkpoint.Decision{Approved: true}); err != nil { - t.Fatalf("Resolve without arguments: %v", err) + if resolveErr := s.Resolve(ctx, cp2.ID, checkpoint.Decision{Approved: true}); resolveErr != nil { + t.Fatalf("Resolve without arguments: %v", resolveErr) } got2, err := s.GetCheckpoint(ctx, cp2.ID) if err != nil || got2.Decision == nil { diff --git a/suspension/suspension.go b/suspension/suspension.go index 5c2125d1..a2695548 100644 --- a/suspension/suspension.go +++ b/suspension/suspension.go @@ -39,7 +39,7 @@ const ( // ledger; nobody else may. ReasonAgentReply SuspendReason = "agent_reply" // ReasonDeferredModelCall means the model client deferred the run's - // next model call (cortex.DeferredModelCall). The suspension has no + // next model call (cortex.DeferredModelCallError). The suspension has no // pending calls, its continuation is the state from before that call, // and only a decision on its checkpoint continues it, by making the // same call. From afb78dd546a9ba448b09b4597394cf264a14e705 Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Fri, 13 Nov 2026 19:21:11 -0600 Subject: [PATCH 8/9] build(deps): move to grove v1.10.3 --- a2aremote/go.mod | 4 ++-- a2aremote/go.sum | 8 ++++---- api/go.mod | 4 ++-- api/go.sum | 8 ++++---- extension/go.mod | 8 ++++---- extension/go.sum | 16 ++++++++-------- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- integrations/fabriq/go.mod | 4 ++-- integrations/fabriq/go.sum | 8 ++++---- integrations/sandbox/go.mod | 4 ++-- integrations/sandbox/go.sum | 8 ++++---- inttest/go.mod | 10 +++++----- inttest/go.sum | 20 ++++++++++---------- optimization/go.mod | 8 ++++---- optimization/go.sum | 16 ++++++++-------- sentinel/go.mod | 4 ++-- sentinel/go.sum | 8 ++++---- 18 files changed, 81 insertions(+), 81 deletions(-) diff --git a/a2aremote/go.mod b/a2aremote/go.mod index 25da01cd..e71196f8 100644 --- a/a2aremote/go.mod +++ b/a2aremote/go.mod @@ -7,8 +7,8 @@ replace github.com/xraph/cortex => ../ require ( github.com/stretchr/testify v1.12.1 github.com/xraph/cortex v1.6.1 - github.com/xraph/grove v1.10.2 - github.com/xraph/grove/drivers/sqlitedriver v1.10.2 + github.com/xraph/grove v1.10.3 + github.com/xraph/grove/drivers/sqlitedriver v1.10.3 ) require ( diff --git a/a2aremote/go.sum b/a2aremote/go.sum index 1fa99730..52e43405 100644 --- a/a2aremote/go.sum +++ b/a2aremote/go.sum @@ -31,10 +31,10 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/xraph/shield v1.10.2 h1:LJlnTeXvDov+GBPhOGsnyoiH/1Zsx5gaOPfNfSAGoxU= github.com/xraph/shield v1.10.2/go.mod h1:xkWUxiFghxQV74UVNfDsSz/mCjQ18mpt4W4kxGiBqJU= github.com/xraph/vessel v1.0.5 h1:4In3UmIJzOrym5+FXE/uvm1jR+gofUqVmEjKjjgE3gw= diff --git a/api/go.mod b/api/go.mod index ac6f9bf9..a42d4b40 100644 --- a/api/go.mod +++ b/api/go.mod @@ -9,8 +9,8 @@ require ( github.com/xraph/cortex v1.8.1-0.20261025225257-d0eb82f6ed68 github.com/xraph/cortex/optimization v0.0.0-20261026160546-1b6f6bc9f367 github.com/xraph/forge v1.13.6 - github.com/xraph/grove v1.10.2 - github.com/xraph/grove/drivers/sqlitedriver v1.10.2 + github.com/xraph/grove v1.10.3 + github.com/xraph/grove/drivers/sqlitedriver v1.10.3 ) require ( diff --git a/api/go.sum b/api/go.sum index 547979e7..6e06fe4e 100644 --- a/api/go.sum +++ b/api/go.sum @@ -291,10 +291,10 @@ github.com/xraph/forge v1.13.6 h1:JCLMd1eEd7Ay96iVg8tFlYDBMyVO8ltZSIMGRL7ktko= github.com/xraph/forge v1.13.6/go.mod h1:gg/KN2RcPOZl0DMxLRp7QQYBW3wLscXgYdUzfcpWDeI= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/xraph/shield v1.10.2 h1:LJlnTeXvDov+GBPhOGsnyoiH/1Zsx5gaOPfNfSAGoxU= github.com/xraph/shield v1.10.2/go.mod h1:xkWUxiFghxQV74UVNfDsSz/mCjQ18mpt4W4kxGiBqJU= github.com/xraph/vessel v1.0.5 h1:4In3UmIJzOrym5+FXE/uvm1jR+gofUqVmEjKjjgE3gw= diff --git a/extension/go.mod b/extension/go.mod index a97a7018..ea96a7e7 100644 --- a/extension/go.mod +++ b/extension/go.mod @@ -13,8 +13,8 @@ require ( github.com/xraph/cortex/sentinel v0.0.0-20261026122752-0b00c24b30ca github.com/xraph/forge v1.13.6 github.com/xraph/go-utils v1.5.0 - github.com/xraph/grove v1.10.2 - github.com/xraph/grove/drivers/sqlitedriver v1.10.2 + github.com/xraph/grove v1.10.3 + github.com/xraph/grove/drivers/sqlitedriver v1.10.3 github.com/xraph/nexus v1.9.3 github.com/xraph/shield v1.10.2 github.com/xraph/vessel v1.0.5 @@ -107,8 +107,8 @@ require ( github.com/xdg-go/scram v1.4.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/xraph/confy v1.1.0 // indirect - github.com/xraph/grove/drivers/mongodriver v1.10.2 // indirect - github.com/xraph/grove/drivers/pgdriver v1.10.2 // indirect + github.com/xraph/grove/drivers/mongodriver v1.10.3 // indirect + github.com/xraph/grove/drivers/pgdriver v1.10.3 // indirect github.com/xraph/sentinel v1.10.1 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect go.jetify.com/typeid/v2 v2.0.0-alpha.3 // indirect diff --git a/extension/go.sum b/extension/go.sum index 6be53450..efd8e522 100644 --- a/extension/go.sum +++ b/extension/go.sum @@ -378,14 +378,14 @@ github.com/xraph/forge v1.13.6 h1:JCLMd1eEd7Ay96iVg8tFlYDBMyVO8ltZSIMGRL7ktko= github.com/xraph/forge v1.13.6/go.mod h1:gg/KN2RcPOZl0DMxLRp7QQYBW3wLscXgYdUzfcpWDeI= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/mongodriver v1.10.2 h1:IrIqWzg72iL6qroh/llft+CH0FygMS2wyBwYkVeNQ2s= -github.com/xraph/grove/drivers/mongodriver v1.10.2/go.mod h1:x/t82VUmjCAuUluOHHfDrJCHdGKtd3M2pnKvBnUxvwU= -github.com/xraph/grove/drivers/pgdriver v1.10.2 h1:LMMqPMPtj1OgA1npfqOWsknMQZbJ7kFwMrdrj32Mp3I= -github.com/xraph/grove/drivers/pgdriver v1.10.2/go.mod h1:aV31nDRM4Igx4vqwjM9u8v0A1URL8YyDyXCuABLOuPA= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/mongodriver v1.10.3 h1:hk6xPlpDKeBFfp9MfFVdzValnITY5lG1ypseBwzIaJo= +github.com/xraph/grove/drivers/mongodriver v1.10.3/go.mod h1:x/t82VUmjCAuUluOHHfDrJCHdGKtd3M2pnKvBnUxvwU= +github.com/xraph/grove/drivers/pgdriver v1.10.3 h1:firu2F10BdN2ciaYmFw1MrQ4z0DKnNQz4ifg0gLzxes= +github.com/xraph/grove/drivers/pgdriver v1.10.3/go.mod h1:aV31nDRM4Igx4vqwjM9u8v0A1URL8YyDyXCuABLOuPA= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/xraph/nexus v1.9.3 h1:swjUBnkGFYs6f0sifyW7a7CmHetwjmslUKGsH19CFhQ= github.com/xraph/nexus v1.9.3/go.mod h1:0hU8ddSNwAw1bh7HlAe9v4ftR0VR0FlYZqpRjKAuhT0= github.com/xraph/sentinel v1.10.1 h1:zR/qLzkGgbTHOxqm4uxMmR7Ep4dAIq/890+BUA3kQec= diff --git a/go.mod b/go.mod index d363053f..f97a5f5d 100644 --- a/go.mod +++ b/go.mod @@ -10,10 +10,10 @@ require ( github.com/testcontainers/testcontainers-go/modules/mongodb v0.42.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 github.com/xraph/go-utils v1.5.0 - github.com/xraph/grove v1.10.2 - github.com/xraph/grove/drivers/mongodriver v1.10.2 - github.com/xraph/grove/drivers/pgdriver v1.10.2 - github.com/xraph/grove/drivers/sqlitedriver v1.10.2 + github.com/xraph/grove v1.10.3 + github.com/xraph/grove/drivers/mongodriver v1.10.3 + github.com/xraph/grove/drivers/pgdriver v1.10.3 + github.com/xraph/grove/drivers/sqlitedriver v1.10.3 github.com/xraph/nexus v1.9.3 github.com/xraph/shield v1.10.2 github.com/xraph/vessel v1.0.5 diff --git a/go.sum b/go.sum index 2782d977..0cb84821 100644 --- a/go.sum +++ b/go.sum @@ -144,14 +144,14 @@ github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/mongodriver v1.10.2 h1:IrIqWzg72iL6qroh/llft+CH0FygMS2wyBwYkVeNQ2s= -github.com/xraph/grove/drivers/mongodriver v1.10.2/go.mod h1:x/t82VUmjCAuUluOHHfDrJCHdGKtd3M2pnKvBnUxvwU= -github.com/xraph/grove/drivers/pgdriver v1.10.2 h1:LMMqPMPtj1OgA1npfqOWsknMQZbJ7kFwMrdrj32Mp3I= -github.com/xraph/grove/drivers/pgdriver v1.10.2/go.mod h1:aV31nDRM4Igx4vqwjM9u8v0A1URL8YyDyXCuABLOuPA= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/mongodriver v1.10.3 h1:hk6xPlpDKeBFfp9MfFVdzValnITY5lG1ypseBwzIaJo= +github.com/xraph/grove/drivers/mongodriver v1.10.3/go.mod h1:x/t82VUmjCAuUluOHHfDrJCHdGKtd3M2pnKvBnUxvwU= +github.com/xraph/grove/drivers/pgdriver v1.10.3 h1:firu2F10BdN2ciaYmFw1MrQ4z0DKnNQz4ifg0gLzxes= +github.com/xraph/grove/drivers/pgdriver v1.10.3/go.mod h1:aV31nDRM4Igx4vqwjM9u8v0A1URL8YyDyXCuABLOuPA= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/xraph/nexus v1.9.3 h1:swjUBnkGFYs6f0sifyW7a7CmHetwjmslUKGsH19CFhQ= github.com/xraph/nexus v1.9.3/go.mod h1:0hU8ddSNwAw1bh7HlAe9v4ftR0VR0FlYZqpRjKAuhT0= github.com/xraph/shield v1.10.2 h1:LJlnTeXvDov+GBPhOGsnyoiH/1Zsx5gaOPfNfSAGoxU= diff --git a/integrations/fabriq/go.mod b/integrations/fabriq/go.mod index f038dcaf..9061f6be 100644 --- a/integrations/fabriq/go.mod +++ b/integrations/fabriq/go.mod @@ -8,8 +8,8 @@ require ( github.com/xraph/cortex v1.8.1-0.20261025225257-d0eb82f6ed68 github.com/xraph/fabriq/core v1.10.2 github.com/xraph/go-utils v1.5.0 - github.com/xraph/grove v1.10.2 - github.com/xraph/grove/drivers/sqlitedriver v1.10.2 + github.com/xraph/grove v1.10.3 + github.com/xraph/grove/drivers/sqlitedriver v1.10.3 github.com/xraph/vessel v1.0.5 ) diff --git a/integrations/fabriq/go.sum b/integrations/fabriq/go.sum index 00b75d7b..893beeff 100644 --- a/integrations/fabriq/go.sum +++ b/integrations/fabriq/go.sum @@ -36,10 +36,10 @@ github.com/xraph/fabriq/core v1.10.2 h1:06c5vgMMPB9Dmeoqjca5E3vzuPoC8Q6V3gISN1HO github.com/xraph/fabriq/core v1.10.2/go.mod h1:GGSFmRNOHObOJLOwITpmgw8YlP3ylxNDmlR3eoW16FE= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/xraph/shield v1.10.2 h1:LJlnTeXvDov+GBPhOGsnyoiH/1Zsx5gaOPfNfSAGoxU= github.com/xraph/shield v1.10.2/go.mod h1:xkWUxiFghxQV74UVNfDsSz/mCjQ18mpt4W4kxGiBqJU= github.com/xraph/vessel v1.0.5 h1:4In3UmIJzOrym5+FXE/uvm1jR+gofUqVmEjKjjgE3gw= diff --git a/integrations/sandbox/go.mod b/integrations/sandbox/go.mod index 6fb7d31f..853dac03 100644 --- a/integrations/sandbox/go.mod +++ b/integrations/sandbox/go.mod @@ -10,8 +10,8 @@ require ( github.com/opencontainers/runtime-spec v1.3.0 github.com/stretchr/testify v1.12.1 github.com/xraph/cortex v1.8.0 - github.com/xraph/grove v1.10.2 - github.com/xraph/grove/drivers/sqlitedriver v1.10.2 + github.com/xraph/grove v1.10.3 + github.com/xraph/grove/drivers/sqlitedriver v1.10.3 golang.org/x/sys v0.49.0 modernc.org/sqlite v1.58.0 ) diff --git a/integrations/sandbox/go.sum b/integrations/sandbox/go.sum index b0c1d580..53e0629f 100644 --- a/integrations/sandbox/go.sum +++ b/integrations/sandbox/go.sum @@ -115,10 +115,10 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.jetify.com/typeid/v2 v2.0.0-alpha.3 h1:T6RPx6bNl10lp0JN2Xz/XcgLZWSlVmL58Xqy9cgTCcc= diff --git a/inttest/go.mod b/inttest/go.mod index 2237945e..69a5a469 100644 --- a/inttest/go.mod +++ b/inttest/go.mod @@ -15,7 +15,7 @@ require ( github.com/xraph/cortex/integrations/fabriq v0.0.0-20260827002006-bb2528ddd778 github.com/xraph/fabriq v1.10.2 github.com/xraph/fabriq/core v1.10.2 - github.com/xraph/grove/drivers/sqlitedriver v1.10.2 + github.com/xraph/grove/drivers/sqlitedriver v1.10.3 ) require ( @@ -118,10 +118,10 @@ require ( github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xraph/cortex/optimization v0.0.0-20261025141439-37b94d6cbe98 github.com/xraph/go-utils v1.5.0 // indirect - github.com/xraph/grove v1.10.2 - github.com/xraph/grove/drivers/pgdriver v1.10.2 // indirect - github.com/xraph/grove/kv v1.10.2 // indirect - github.com/xraph/grove/kv/drivers/redisdriver v1.10.2 // indirect + github.com/xraph/grove v1.10.3 + github.com/xraph/grove/drivers/pgdriver v1.10.3 // indirect + github.com/xraph/grove/kv v1.10.3 // indirect + github.com/xraph/grove/kv/drivers/redisdriver v1.10.3 // indirect github.com/xraph/trove v1.8.2 // indirect github.com/xraph/vessel v1.0.5 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect diff --git a/inttest/go.sum b/inttest/go.sum index ada20ae9..589efd6c 100644 --- a/inttest/go.sum +++ b/inttest/go.sum @@ -232,16 +232,16 @@ github.com/xraph/fabriq/core v1.10.2 h1:06c5vgMMPB9Dmeoqjca5E3vzuPoC8Q6V3gISN1HO github.com/xraph/fabriq/core v1.10.2/go.mod h1:GGSFmRNOHObOJLOwITpmgw8YlP3ylxNDmlR3eoW16FE= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/pgdriver v1.10.2 h1:LMMqPMPtj1OgA1npfqOWsknMQZbJ7kFwMrdrj32Mp3I= -github.com/xraph/grove/drivers/pgdriver v1.10.2/go.mod h1:aV31nDRM4Igx4vqwjM9u8v0A1URL8YyDyXCuABLOuPA= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= -github.com/xraph/grove/kv v1.10.2 h1:3ic9NWFSGLmcxmy+eHw9t0HrU4WCGnTsSdn56rabS1U= -github.com/xraph/grove/kv v1.10.2/go.mod h1:qW3j9k8OV14itMSQTLpC/INh35HL5ppWRknzUNas2/o= -github.com/xraph/grove/kv/drivers/redisdriver v1.10.2 h1:o82/hCBDJlFmu9ICozAcrnc8piObjN0W7rqTHX6MygE= -github.com/xraph/grove/kv/drivers/redisdriver v1.10.2/go.mod h1:4mAi3I+9Li2zNaBXa2TGUf9aGHlGAAvdbE6WdkfqyZg= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/pgdriver v1.10.3 h1:firu2F10BdN2ciaYmFw1MrQ4z0DKnNQz4ifg0gLzxes= +github.com/xraph/grove/drivers/pgdriver v1.10.3/go.mod h1:aV31nDRM4Igx4vqwjM9u8v0A1URL8YyDyXCuABLOuPA= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove/kv v1.10.3 h1:bnjKdhg/exUqg+YqdwlfT0Ud/Vp+THTHTAJwBTRG8GM= +github.com/xraph/grove/kv v1.10.3/go.mod h1:qW3j9k8OV14itMSQTLpC/INh35HL5ppWRknzUNas2/o= +github.com/xraph/grove/kv/drivers/redisdriver v1.10.3 h1:0dJYfizIMxM1lOgdFSAbf4sCLEN/+Fm4y9fFDnVsN7c= +github.com/xraph/grove/kv/drivers/redisdriver v1.10.3/go.mod h1:4mAi3I+9Li2zNaBXa2TGUf9aGHlGAAvdbE6WdkfqyZg= github.com/xraph/shield v1.10.2 h1:LJlnTeXvDov+GBPhOGsnyoiH/1Zsx5gaOPfNfSAGoxU= github.com/xraph/shield v1.10.2/go.mod h1:xkWUxiFghxQV74UVNfDsSz/mCjQ18mpt4W4kxGiBqJU= github.com/xraph/trove v1.8.2 h1:rPe55n9keHQgQ3iSmCyiv2JNfjjlE0yozqeARmeIz8Q= diff --git a/optimization/go.mod b/optimization/go.mod index f93c4d47..aee75f1e 100644 --- a/optimization/go.mod +++ b/optimization/go.mod @@ -6,10 +6,10 @@ require ( github.com/testcontainers/testcontainers-go/modules/mongodb v0.42.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 github.com/xraph/cortex v1.8.1-0.20261025225257-d0eb82f6ed68 - github.com/xraph/grove v1.10.2 - github.com/xraph/grove/drivers/mongodriver v1.10.2 - github.com/xraph/grove/drivers/pgdriver v1.10.2 - github.com/xraph/grove/drivers/sqlitedriver v1.10.2 + github.com/xraph/grove v1.10.3 + github.com/xraph/grove/drivers/mongodriver v1.10.3 + github.com/xraph/grove/drivers/pgdriver v1.10.3 + github.com/xraph/grove/drivers/sqlitedriver v1.10.3 go.jetify.com/typeid/v2 v2.0.0-alpha.3 go.mongodb.org/mongo-driver/v2 v2.10.0 ) diff --git a/optimization/go.sum b/optimization/go.sum index e610ee57..f6bd2b9c 100644 --- a/optimization/go.sum +++ b/optimization/go.sum @@ -139,14 +139,14 @@ github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6 github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/mongodriver v1.10.2 h1:IrIqWzg72iL6qroh/llft+CH0FygMS2wyBwYkVeNQ2s= -github.com/xraph/grove/drivers/mongodriver v1.10.2/go.mod h1:x/t82VUmjCAuUluOHHfDrJCHdGKtd3M2pnKvBnUxvwU= -github.com/xraph/grove/drivers/pgdriver v1.10.2 h1:LMMqPMPtj1OgA1npfqOWsknMQZbJ7kFwMrdrj32Mp3I= -github.com/xraph/grove/drivers/pgdriver v1.10.2/go.mod h1:aV31nDRM4Igx4vqwjM9u8v0A1URL8YyDyXCuABLOuPA= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/mongodriver v1.10.3 h1:hk6xPlpDKeBFfp9MfFVdzValnITY5lG1ypseBwzIaJo= +github.com/xraph/grove/drivers/mongodriver v1.10.3/go.mod h1:x/t82VUmjCAuUluOHHfDrJCHdGKtd3M2pnKvBnUxvwU= +github.com/xraph/grove/drivers/pgdriver v1.10.3 h1:firu2F10BdN2ciaYmFw1MrQ4z0DKnNQz4ifg0gLzxes= +github.com/xraph/grove/drivers/pgdriver v1.10.3/go.mod h1:aV31nDRM4Igx4vqwjM9u8v0A1URL8YyDyXCuABLOuPA= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/xraph/shield v1.10.2 h1:LJlnTeXvDov+GBPhOGsnyoiH/1Zsx5gaOPfNfSAGoxU= github.com/xraph/shield v1.10.2/go.mod h1:xkWUxiFghxQV74UVNfDsSz/mCjQ18mpt4W4kxGiBqJU= github.com/xraph/vessel v1.0.5 h1:4In3UmIJzOrym5+FXE/uvm1jR+gofUqVmEjKjjgE3gw= diff --git a/sentinel/go.mod b/sentinel/go.mod index 28329161..15dca542 100644 --- a/sentinel/go.mod +++ b/sentinel/go.mod @@ -7,8 +7,8 @@ replace github.com/xraph/cortex => ../ require ( github.com/xraph/cortex v1.8.1-0.20261025225257-d0eb82f6ed68 github.com/xraph/cortex/optimization v0.0.0-20261026160546-1b6f6bc9f367 - github.com/xraph/grove v1.10.2 - github.com/xraph/grove/drivers/sqlitedriver v1.10.2 + github.com/xraph/grove v1.10.3 + github.com/xraph/grove/drivers/sqlitedriver v1.10.3 github.com/xraph/sentinel v1.10.1 github.com/xraph/vessel v1.0.5 ) diff --git a/sentinel/go.sum b/sentinel/go.sum index c78ff8de..94509ba1 100644 --- a/sentinel/go.sum +++ b/sentinel/go.sum @@ -291,10 +291,10 @@ github.com/xraph/forge v1.13.6 h1:JCLMd1eEd7Ay96iVg8tFlYDBMyVO8ltZSIMGRL7ktko= github.com/xraph/forge v1.13.6/go.mod h1:gg/KN2RcPOZl0DMxLRp7QQYBW3wLscXgYdUzfcpWDeI= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/xraph/sentinel v1.10.1 h1:zR/qLzkGgbTHOxqm4uxMmR7Ep4dAIq/890+BUA3kQec= github.com/xraph/sentinel v1.10.1/go.mod h1:vuTrYQUVHVxCOK6JScuSp9cPfMpwNXYouXvIWZmVRQs= github.com/xraph/shield v1.10.2 h1:LJlnTeXvDov+GBPhOGsnyoiH/1Zsx5gaOPfNfSAGoxU= From d6e4f7b982b8648ad611cff9aecdfd51dc8ad203 Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Fri, 13 Nov 2026 19:35:00 -0600 Subject: [PATCH 9/9] build(deps): tidy mcpbridge and grpcbind for grove v1.10.3 --- a2aremote/grpcbind/go.sum | 8 ++++---- mcpbridge/go.sum | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/a2aremote/grpcbind/go.sum b/a2aremote/grpcbind/go.sum index c8b76f21..58e5e60a 100644 --- a/a2aremote/grpcbind/go.sum +++ b/a2aremote/grpcbind/go.sum @@ -29,10 +29,10 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/xraph/shield v1.10.2 h1:LJlnTeXvDov+GBPhOGsnyoiH/1Zsx5gaOPfNfSAGoxU= github.com/xraph/shield v1.10.2/go.mod h1:xkWUxiFghxQV74UVNfDsSz/mCjQ18mpt4W4kxGiBqJU= github.com/xraph/vessel v1.0.5 h1:4In3UmIJzOrym5+FXE/uvm1jR+gofUqVmEjKjjgE3gw= diff --git a/mcpbridge/go.sum b/mcpbridge/go.sum index 6570d7cc..f7fa7de1 100644 --- a/mcpbridge/go.sum +++ b/mcpbridge/go.sum @@ -35,10 +35,10 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/xraph/go-utils v1.5.0 h1:44pIJooK565ds/Ica0yyPoRNbr0qUTgj1aay8ewad1w= github.com/xraph/go-utils v1.5.0/go.mod h1:fMahP6kPUjJP5UmzVje2GSL7OGT9+xpYkukK2dLjSMM= -github.com/xraph/grove v1.10.2 h1:3ZoKJpU21gIv8p98RjQE4wFcAhesXvTqSSqrroNOXaI= -github.com/xraph/grove v1.10.2/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2 h1:YLB9bGSxJnIquPN31etQGtrCWQfuuqfbcjk4eCkPT9Y= -github.com/xraph/grove/drivers/sqlitedriver v1.10.2/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= +github.com/xraph/grove v1.10.3 h1:bTVuW6le1hIBLYzpmPBgcVO826FcncYYFFDJntZT7DY= +github.com/xraph/grove v1.10.3/go.mod h1:oQ5d+MiSDkDxiRCKBOlmieJmzFXvanY/kk9VfvQO40w= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3 h1:a7f/Ir/4iMQsE+EDshgOtUf+Vpu2EIhOMsVDlONveeI= +github.com/xraph/grove/drivers/sqlitedriver v1.10.3/go.mod h1:UlTGQQRp7VD4Rjj8HN6XjW1TMZl0iSNI4LApoQnChYQ= github.com/xraph/shield v1.10.2 h1:LJlnTeXvDov+GBPhOGsnyoiH/1Zsx5gaOPfNfSAGoxU= github.com/xraph/shield v1.10.2/go.mod h1:xkWUxiFghxQV74UVNfDsSz/mCjQ18mpt4W4kxGiBqJU= github.com/xraph/vessel v1.0.5 h1:4In3UmIJzOrym5+FXE/uvm1jR+gofUqVmEjKjjgE3gw=