Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions engine/continue_amend_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ func TestContinueRun_amendedArgumentsReplaceTheModelsArguments(t *testing.T) {
require.JSONEq(t, `{"to":"b"}`, call.Arguments)
require.Equal(t, true, call.Metadata["arguments_amended"])
require.Equal(t, "bob", call.Metadata["amended_by"])
require.Equal(t, "call-1", call.Metadata[run.MetadataCallID], "the edit must not drop the call id")
}
}
}
Expand Down
13 changes: 13 additions & 0 deletions engine/react.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,17 @@ func stateFromContinuation(cont suspension.Continuation) *reactState {
}
}

// callMetadata is a tool call row's metadata naming the model's id for
// the call, nil when the model gave none. The id is generated text, so a
// run that requires verification keeps it private like the rest.
func callMetadata(r *run.Run, callID string) map[string]any {
callID = publicGenerationText(r, callID)
if callID == "" {
return nil
}
return map[string]any{run.MetadataCallID: callID}
}

// runReAct executes an agent using the ReAct reasoning loop synchronously.
func (e *Engine) runReAct(ctx context.Context, ag *agent.Config, input string, overrides *RunOverrides) (*run.Run, error) {
p, err := e.prepareAgent(ctx, ag, overrides)
Expand Down Expand Up @@ -427,6 +438,7 @@ func (e *Engine) continueReAct(ctx context.Context, ag *agent.Config, cfg resolv
Error: publicToolFailure(ctx, outcome),
StartedAt: &tcStart,
CompletedAt: &tcEnd,
Metadata: callMetadata(r, tc.ID),
}
if err := e.store.CreateToolCall(ctx, toolCall); err != nil {
e.logError("create tool call", err, log.String("run_id", r.ID.String()))
Expand Down Expand Up @@ -946,6 +958,7 @@ func (e *Engine) continueStreamReAct(ctx context.Context, ag *agent.Config, cfg
Error: publicToolFailure(ctx, outcome),
StartedAt: &tcStart,
CompletedAt: &tcEnd,
Metadata: callMetadata(r, tc.ID),
}
if err := e.store.CreateToolCall(ctx, toolCall); err != nil {
e.logError("create tool call", err, log.String("run_id", r.ID.String()))
Expand Down
1 change: 1 addition & 0 deletions engine/reclaim.go
Original file line number Diff line number Diff line change
Expand Up @@ -486,6 +486,7 @@ func (e *Engine) replayStep(ctx context.Context, ag *agent.Config, r *run.Run, s
Error: publicToolFailure(ctx, outcome),
StartedAt: &startedAt,
CompletedAt: &completedAt,
Metadata: callMetadata(r, call.ID),
}
if err := e.store.CreateToolCall(ctx, row); err != nil {
e.logError("create replayed tool call", err, log.String("run_id", r.ID.String()))
Expand Down
14 changes: 14 additions & 0 deletions engine/reclaim_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,20 @@ func TestContinueReclaimedRunReplaysOnlyMissingCalls(t *testing.T) {
require.Equal(t, "a-result", tools[0].Content)
require.Equal(t, "call-b", tools[1].ToolCallID)
require.Equal(t, "b-result", tools[1].Content)

// Both rows name the model's call: a's from the first worker's loop,
// b's from the reclaimer's replay.
steps, err := f.store.ListSteps(leaseScope(), r.ID)
require.NoError(t, err)
callIDs := map[string]any{}
for _, step := range steps {
rows, listErr := f.store.ListToolCalls(leaseScope(), step.ID)
require.NoError(t, listErr)
for _, row := range rows {
callIDs[row.ToolName] = row.Metadata[run.MetadataCallID]
}
}
require.Equal(t, map[string]any{"a": "call-a", "b": "call-b"}, callIDs)
}

func TestContinueReclaimedRunFailsUncertainDispatch(t *testing.T) {
Expand Down
7 changes: 6 additions & 1 deletion engine/resume.go
Original file line number Diff line number Diff line change
Expand Up @@ -817,9 +817,14 @@ func (e *Engine) recordResumedCalls(ctx context.Context, ag *agent.Config, r *ru
// hide exactly the latency an external tool introduces.
StartedAt: &startedAt,
CompletedAt: &completedAt,
Metadata: callMetadata(r, p.ID),
}
if res.Execute && len(res.Arguments) > 0 {
toolCall.Metadata = map[string]any{"arguments_amended": true, "amended_by": res.AmendedBy}
if toolCall.Metadata == nil {
toolCall.Metadata = map[string]any{}
}
toolCall.Metadata["arguments_amended"] = true
toolCall.Metadata["amended_by"] = res.AmendedBy
}
if err := e.store.CreateToolCall(ctx, toolCall); err != nil {
e.logError("create resumed tool call", err, log.String("run_id", r.ID.String()))
Expand Down
138 changes: 138 additions & 0 deletions engine/tool_call_id_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
package engine

import (
"context"
"encoding/json"
"testing"

"github.com/stretchr/testify/require"

"github.com/xraph/cortex"
"github.com/xraph/cortex/checkpoint"
"github.com/xraph/cortex/id"
"github.com/xraph/cortex/llm"
"github.com/xraph/cortex/run"
)

// plainLookupTool is an ungated tool: the fixture's authorizer escalates only
// the gated one, so a call to this one runs inside the agent loop.
func plainLookupTool() llm.Tool {
return llm.Tool{Name: "lookup", Description: "test-only ungated tool"}
}

func newLookupFixture(t *testing.T, calls []llm.ToolCall, opts ...Option) *continueFixture {
t.Helper()
ctx := cortexScopeContext()
opts = append(opts, WithTool(plainLookupTool(), func(context.Context, cortex.Invocation) (string, error) {
return "found", nil
}))
return buildContinueFixture(ctx, t, newApprovalStore(ctx, t), calls, opts...)
}

// toolCallRows is every tool call row of a run, in step order.
func toolCallRows(t *testing.T, f *continueFixture, runID id.AgentRunID) []*run.ToolCall {
t.Helper()
steps, err := f.e.ListSteps(f.ctx, runID)
require.NoError(t, err)
var rows []*run.ToolCall
for _, step := range steps {
calls, err := f.e.ListToolCalls(f.ctx, step.ID)
require.NoError(t, err)
rows = append(rows, calls...)
}
return rows
}

// recordedCallIDs is the call id each named tool's rows carry.
func recordedCallIDs(t *testing.T, f *continueFixture, runID id.AgentRunID, toolName string) []any {
t.Helper()
var ids []any
for _, row := range toolCallRows(t, f, runID) {
if row.ToolName == toolName {
ids = append(ids, row.Metadata[run.MetadataCallID])
}
}
return ids
}

func twoLookups() []llm.ToolCall {
return []llm.ToolCall{
{ID: "call-a", Name: plainLookupTool().Name, Arguments: `{"q":"a"}`},
{ID: "call-b", Name: plainLookupTool().Name, Arguments: `{"q":"b"}`},
}
}

func TestRunAgent_recordsEachToolCallsID(t *testing.T) {
f := newLookupFixture(t, twoLookups())
done, err := f.e.RunAgent(as(f.ctx, alice), "assistant", "look up", nil)
require.NoError(t, err)
require.Equal(t, run.StateCompleted, done.State)

require.ElementsMatch(t, []any{"call-a", "call-b"}, recordedCallIDs(t, f, done.ID, plainLookupTool().Name))
}

func TestStreamAgent_recordsEachToolCallsID(t *testing.T) {
f := newLookupFixture(t, twoLookups())
events := make(chan StreamEvent, 64)
require.NoError(t, f.e.StreamAgent(as(f.ctx, alice), "assistant", "look up", nil, events))
for range events { //nolint:revive // drain the stream until the run ends
}

runs, err := f.e.ListRuns(f.ctx, &run.ListFilter{})
require.NoError(t, err)
require.Len(t, runs, 1)
require.Equal(t, run.StateCompleted, runs[0].State)

require.ElementsMatch(t, []any{"call-a", "call-b"}, recordedCallIDs(t, f, runs[0].ID, plainLookupTool().Name))
}

func TestRunAgent_aCallWithNoIDLeavesNoCallIDKey(t *testing.T) {
f := newLookupFixture(t, []llm.ToolCall{{Name: plainLookupTool().Name, Arguments: `{}`}})
done, err := f.e.RunAgent(as(f.ctx, alice), "assistant", "look up", nil)
require.NoError(t, err)
require.Equal(t, run.StateCompleted, done.State)

rows := toolCallRows(t, f, done.ID)
require.Len(t, rows, 1)
require.Nil(t, rows[0].Metadata, "a call the model gave no id must not record an empty one")
}

func TestCallMetadata(t *testing.T) {
public := &run.Run{}
require.Nil(t, callMetadata(public, ""))
require.Equal(t, map[string]any{run.MetadataCallID: "call-1"}, callMetadata(public, "call-1"))

// A run that requires verification keeps the model's id private, as
// it does the rest of what the model generated.
private := &run.Run{Provenance: run.ExecutionProvenance{VerificationRequired: true}}
require.Nil(t, callMetadata(private, "call-1"))
}

func TestContinueRun_approvedCallRecordsItsCallID(t *testing.T) {
f := newAmendFixture(t)
paused, cp := f.pause(t)
require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{Approved: true, DecidedBy: "bob"}))
done, err := f.e.ContinueRun(f.ctx, paused.ID)
require.NoError(t, err)
require.Equal(t, run.StateCompleted, done.State)

rows := toolCallRows(t, f, paused.ID)
require.Len(t, rows, 1)
require.Equal(t, map[string]any{run.MetadataCallID: "call-1"}, rows[0].Metadata, "an unamended call records only its id")
}

func TestContinueRun_rejectedCallRecordsItsCallID(t *testing.T) {
f := newAmendFixture(t, WithRejectionsReturnToModel())
paused, cp := f.pause(t)
require.NoError(t, f.e.DecideCheckpoint(as(f.ctx, bob), cp.ID, checkpoint.Decision{
DecidedBy: "bob",
Reason: "no",
AmendedArguments: json.RawMessage(`{"to":"b"}`),
}))
_, err := f.e.ContinueRun(f.ctx, paused.ID)
require.NoError(t, err)

rows := toolCallRows(t, f, paused.ID)
require.Len(t, rows, 1)
require.Equal(t, map[string]any{run.MetadataCallID: "call-1"}, rows[0].Metadata, "a rejection records the id and no edit")
}
6 changes: 6 additions & 0 deletions run/toolcall.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ import (
"github.com/xraph/cortex/id"
)

// MetadataCallID is the tool call metadata key that holds the model's id
// for the call (llm.ToolCall.ID, suspension.PendingCall.ID): the id its
// tool result and an approval decision name it by. It is not unique
// across a run: a later model response may reuse it.
const MetadataCallID = "call_id"

// ToolCall represents a single tool invocation within a step.
type ToolCall struct {
cortex.Entity
Expand Down
Loading