From 8a7f18b5482ac6dcb7fafaf0b863112b3c3e6245 Mon Sep 17 00:00:00 2001 From: Rex Raphael Date: Wed, 18 Nov 2026 12:45:45 -0600 Subject: [PATCH] feat(run): record the model's call id on each tool call --- engine/continue_amend_test.go | 1 + engine/react.go | 13 ++++ engine/reclaim.go | 1 + engine/reclaim_test.go | 14 ++++ engine/resume.go | 7 +- engine/tool_call_id_test.go | 138 ++++++++++++++++++++++++++++++++++ run/toolcall.go | 6 ++ 7 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 engine/tool_call_id_test.go diff --git a/engine/continue_amend_test.go b/engine/continue_amend_test.go index 7e9b4983..d4036495 100644 --- a/engine/continue_amend_test.go +++ b/engine/continue_amend_test.go @@ -78,6 +78,7 @@ func TestContinueRun_amendedArgumentsReplaceTheModelsArguments(t *testing.T) { require.JSONEq(t, `{"to":"b"}`, call.Arguments) require.Equal(t, true, call.Metadata["arguments_amended"]) require.Equal(t, "bob", call.Metadata["amended_by"]) + require.Equal(t, "call-1", call.Metadata[run.MetadataCallID], "the edit must not drop the call id") } } } diff --git a/engine/react.go b/engine/react.go index cf38600b..8e96dc59 100644 --- a/engine/react.go +++ b/engine/react.go @@ -125,6 +125,17 @@ func stateFromContinuation(cont suspension.Continuation) *reactState { } } +// callMetadata is a tool call row's metadata naming the model's id for +// the call, nil when the model gave none. The id is generated text, so a +// run that requires verification keeps it private like the rest. +func callMetadata(r *run.Run, callID string) map[string]any { + callID = publicGenerationText(r, callID) + if callID == "" { + return nil + } + return map[string]any{run.MetadataCallID: callID} +} + // runReAct executes an agent using the ReAct reasoning loop synchronously. func (e *Engine) runReAct(ctx context.Context, ag *agent.Config, input string, overrides *RunOverrides) (*run.Run, error) { p, err := e.prepareAgent(ctx, ag, overrides) @@ -427,6 +438,7 @@ func (e *Engine) continueReAct(ctx context.Context, ag *agent.Config, cfg resolv Error: publicToolFailure(ctx, outcome), StartedAt: &tcStart, CompletedAt: &tcEnd, + Metadata: callMetadata(r, tc.ID), } if err := e.store.CreateToolCall(ctx, toolCall); err != nil { e.logError("create tool call", err, log.String("run_id", r.ID.String())) @@ -946,6 +958,7 @@ func (e *Engine) continueStreamReAct(ctx context.Context, ag *agent.Config, cfg Error: publicToolFailure(ctx, outcome), StartedAt: &tcStart, CompletedAt: &tcEnd, + Metadata: callMetadata(r, tc.ID), } if err := e.store.CreateToolCall(ctx, toolCall); err != nil { e.logError("create tool call", err, log.String("run_id", r.ID.String())) diff --git a/engine/reclaim.go b/engine/reclaim.go index d26141a6..bbb46087 100644 --- a/engine/reclaim.go +++ b/engine/reclaim.go @@ -486,6 +486,7 @@ func (e *Engine) replayStep(ctx context.Context, ag *agent.Config, r *run.Run, s Error: publicToolFailure(ctx, outcome), StartedAt: &startedAt, CompletedAt: &completedAt, + Metadata: callMetadata(r, call.ID), } if err := e.store.CreateToolCall(ctx, row); err != nil { e.logError("create replayed tool call", err, log.String("run_id", r.ID.String())) diff --git a/engine/reclaim_test.go b/engine/reclaim_test.go index 4417d94c..cc1c03eb 100644 --- a/engine/reclaim_test.go +++ b/engine/reclaim_test.go @@ -343,6 +343,20 @@ func TestContinueReclaimedRunReplaysOnlyMissingCalls(t *testing.T) { require.Equal(t, "a-result", tools[0].Content) require.Equal(t, "call-b", tools[1].ToolCallID) require.Equal(t, "b-result", tools[1].Content) + + // Both rows name the model's call: a's from the first worker's loop, + // b's from the reclaimer's replay. + steps, err := f.store.ListSteps(leaseScope(), r.ID) + require.NoError(t, err) + callIDs := map[string]any{} + for _, step := range steps { + rows, listErr := f.store.ListToolCalls(leaseScope(), step.ID) + require.NoError(t, listErr) + for _, row := range rows { + callIDs[row.ToolName] = row.Metadata[run.MetadataCallID] + } + } + require.Equal(t, map[string]any{"a": "call-a", "b": "call-b"}, callIDs) } func TestContinueReclaimedRunFailsUncertainDispatch(t *testing.T) { diff --git a/engine/resume.go b/engine/resume.go index 19b3eaca..8e219ab2 100644 --- a/engine/resume.go +++ b/engine/resume.go @@ -817,9 +817,14 @@ func (e *Engine) recordResumedCalls(ctx context.Context, ag *agent.Config, r *ru // hide exactly the latency an external tool introduces. StartedAt: &startedAt, CompletedAt: &completedAt, + Metadata: callMetadata(r, p.ID), } if res.Execute && len(res.Arguments) > 0 { - toolCall.Metadata = map[string]any{"arguments_amended": true, "amended_by": res.AmendedBy} + if toolCall.Metadata == nil { + toolCall.Metadata = map[string]any{} + } + toolCall.Metadata["arguments_amended"] = true + toolCall.Metadata["amended_by"] = res.AmendedBy } if err := e.store.CreateToolCall(ctx, toolCall); err != nil { e.logError("create resumed tool call", err, log.String("run_id", r.ID.String())) diff --git a/engine/tool_call_id_test.go b/engine/tool_call_id_test.go new file mode 100644 index 00000000..685dd757 --- /dev/null +++ b/engine/tool_call_id_test.go @@ -0,0 +1,138 @@ +package engine + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/xraph/cortex" + "github.com/xraph/cortex/checkpoint" + "github.com/xraph/cortex/id" + "github.com/xraph/cortex/llm" + "github.com/xraph/cortex/run" +) + +// plainLookupTool is an ungated tool: the fixture's authorizer escalates only +// the gated one, so a call to this one runs inside the agent loop. +func plainLookupTool() llm.Tool { + return llm.Tool{Name: "lookup", Description: "test-only ungated tool"} +} + +func newLookupFixture(t *testing.T, calls []llm.ToolCall, opts ...Option) *continueFixture { + t.Helper() + ctx := cortexScopeContext() + opts = append(opts, WithTool(plainLookupTool(), func(context.Context, cortex.Invocation) (string, error) { + return "found", nil + })) + return buildContinueFixture(ctx, t, newApprovalStore(ctx, t), calls, opts...) +} + +// toolCallRows is every tool call row of a run, in step order. +func toolCallRows(t *testing.T, f *continueFixture, runID id.AgentRunID) []*run.ToolCall { + t.Helper() + steps, err := f.e.ListSteps(f.ctx, runID) + require.NoError(t, err) + var rows []*run.ToolCall + for _, step := range steps { + calls, err := f.e.ListToolCalls(f.ctx, step.ID) + require.NoError(t, err) + rows = append(rows, calls...) + } + return rows +} + +// recordedCallIDs is the call id each named tool's rows carry. +func recordedCallIDs(t *testing.T, f *continueFixture, runID id.AgentRunID, toolName string) []any { + t.Helper() + var ids []any + for _, row := range toolCallRows(t, f, runID) { + if row.ToolName == toolName { + ids = append(ids, row.Metadata[run.MetadataCallID]) + } + } + return ids +} + +func twoLookups() []llm.ToolCall { + return []llm.ToolCall{ + {ID: "call-a", Name: plainLookupTool().Name, Arguments: `{"q":"a"}`}, + {ID: "call-b", Name: plainLookupTool().Name, Arguments: `{"q":"b"}`}, + } +} + +func TestRunAgent_recordsEachToolCallsID(t *testing.T) { + f := newLookupFixture(t, twoLookups()) + done, err := f.e.RunAgent(as(f.ctx, alice), "assistant", "look up", nil) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, done.State) + + require.ElementsMatch(t, []any{"call-a", "call-b"}, recordedCallIDs(t, f, done.ID, plainLookupTool().Name)) +} + +func TestStreamAgent_recordsEachToolCallsID(t *testing.T) { + f := newLookupFixture(t, twoLookups()) + events := make(chan StreamEvent, 64) + require.NoError(t, f.e.StreamAgent(as(f.ctx, alice), "assistant", "look up", nil, events)) + for range events { //nolint:revive // drain the stream until the run ends + } + + runs, err := f.e.ListRuns(f.ctx, &run.ListFilter{}) + require.NoError(t, err) + require.Len(t, runs, 1) + require.Equal(t, run.StateCompleted, runs[0].State) + + require.ElementsMatch(t, []any{"call-a", "call-b"}, recordedCallIDs(t, f, runs[0].ID, plainLookupTool().Name)) +} + +func TestRunAgent_aCallWithNoIDLeavesNoCallIDKey(t *testing.T) { + f := newLookupFixture(t, []llm.ToolCall{{Name: plainLookupTool().Name, Arguments: `{}`}}) + done, err := f.e.RunAgent(as(f.ctx, alice), "assistant", "look up", nil) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, done.State) + + rows := toolCallRows(t, f, done.ID) + require.Len(t, rows, 1) + require.Nil(t, rows[0].Metadata, "a call the model gave no id must not record an empty one") +} + +func TestCallMetadata(t *testing.T) { + public := &run.Run{} + require.Nil(t, callMetadata(public, "")) + require.Equal(t, map[string]any{run.MetadataCallID: "call-1"}, callMetadata(public, "call-1")) + + // A run that requires verification keeps the model's id private, as + // it does the rest of what the model generated. + private := &run.Run{Provenance: run.ExecutionProvenance{VerificationRequired: true}} + require.Nil(t, callMetadata(private, "call-1")) +} + +func TestContinueRun_approvedCallRecordsItsCallID(t *testing.T) { + f := newAmendFixture(t) + paused, cp := f.pause(t) + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"})) + done, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + require.Equal(t, run.StateCompleted, done.State) + + rows := toolCallRows(t, f, paused.ID) + require.Len(t, rows, 1) + require.Equal(t, map[string]any{run.MetadataCallID: "call-1"}, rows[0].Metadata, "an unamended call records only its id") +} + +func TestContinueRun_rejectedCallRecordsItsCallID(t *testing.T) { + f := newAmendFixture(t, WithRejectionsReturnToModel()) + paused, cp := f.pause(t) + require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{ + DecidedBy: "bob", + Reason: "no", + AmendedArguments: json.RawMessage(`{"to":"b"}`), + })) + _, err := f.e.ContinueRun(f.ctx, paused.ID) + require.NoError(t, err) + + rows := toolCallRows(t, f, paused.ID) + require.Len(t, rows, 1) + require.Equal(t, map[string]any{run.MetadataCallID: "call-1"}, rows[0].Metadata, "a rejection records the id and no edit") +} diff --git a/run/toolcall.go b/run/toolcall.go index 2602d10e..5adf2cd6 100644 --- a/run/toolcall.go +++ b/run/toolcall.go @@ -7,6 +7,12 @@ import ( "github.com/xraph/cortex/id" ) +// MetadataCallID is the tool call metadata key that holds the model's id +// for the call (llm.ToolCall.ID, suspension.PendingCall.ID): the id its +// tool result and an approval decision name it by. It is not unique +// across a run: a later model response may reuse it. +const MetadataCallID = "call_id" + // ToolCall represents a single tool invocation within a step. type ToolCall struct { cortex.Entity