From 7809f6d6308c42afcb5ae51d25a57b4cb0d348e1 Mon Sep 17 00:00:00 2001 From: allocsys <225476909+allocsys@users.noreply.github.com> Date: Thu, 27 Aug 2026 04:05:06 +0330 Subject: [PATCH 1/8] agent_checkpoint.js: thread pendingVerification through save/load, for resuming mid self-verification-pass --- connectors/gemini/agent_checkpoint.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/connectors/gemini/agent_checkpoint.js b/connectors/gemini/agent_checkpoint.js index c9675d4..268b520 100644 --- a/connectors/gemini/agent_checkpoint.js +++ b/connectors/gemini/agent_checkpoint.js @@ -46,7 +46,7 @@ function metaKey(runId) { // - transcript/stepsDone/task/repeatCounts/consecutiveAllRepeatSteps: the // small stuff, always written in full (cheap regardless of run length). // Fails open -- never throws. -export async function saveCheckpoint(runId, { newContents = [], transcript, stepsDone, task, repeatCounts, consecutiveAllRepeatSteps, provider, model, maxOutputTokens }) { +export async function saveCheckpoint(runId, { newContents = [], transcript, stepsDone, task, repeatCounts, consecutiveAllRepeatSteps, provider, model, maxOutputTokens, pendingVerification }) { const client = getRedis(); if (!client) return; try { @@ -59,7 +59,7 @@ export async function saveCheckpoint(runId, { newContents = [], transcript, step // list-creation time. ops.push(client.expire(contentsKey(runId), CHECKPOINT_TTL_SECONDS)); } - const meta = JSON.stringify({ transcript, stepsDone, task, repeatCounts, consecutiveAllRepeatSteps, provider, model, maxOutputTokens }); + const meta = JSON.stringify({ transcript, stepsDone, task, repeatCounts, consecutiveAllRepeatSteps, provider, model, maxOutputTokens, pendingVerification }); ops.push(client.set(metaKey(runId), meta, { ex: CHECKPOINT_TTL_SECONDS })); await Promise.all(ops); } catch { From 7207e6e2a41606040c97b9e17a161b9129ab2001 Mon Sep 17 00:00:00 2001 From: allocsys <225476909+allocsys@users.noreply.github.com> Date: Thu, 27 Aug 2026 04:05:21 +0330 Subject: [PATCH 2/8] agent_delegate.js: add full-read-outranks-narrow-result preamble rule + self-verification-pass mechanism --- connectors/gemini/agent_delegate.js | 33 ++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/connectors/gemini/agent_delegate.js b/connectors/gemini/agent_delegate.js index d74d8ea..680eef1 100644 --- a/connectors/gemini/agent_delegate.js +++ b/connectors/gemini/agent_delegate.js @@ -923,7 +923,38 @@ const SYSTEM_PREAMBLE = "content you fetched (not just the ones that confirm your leaning) and check it against the specific " + "claim in the question -- do not let a majority of confirming sources outvote a single contradicting " + "one you already retrieved. If you find a contradiction this way, quote it and flag it explicitly " + - "even if most of what you found points the other way."; + "even if most of what you found points the other way.\n\n" + + "IMPORTANT -- a full/direct read outranks a narrower or derived result for the SAME fact: when a " + + "complete, direct read of a file or page (github_read_file, github_get_file_at_commit, notion_get_page, " + + "etc.) and a narrower or derived result about the same thing (a github_search_code snippet, a grep hit, " + + "a mem0_search match) disagree, trust the full/direct read -- it is the more authoritative source, even " + + "if the narrower result was fetched more recently in this conversation. A search snippet only shows the " + + "matching line(s) out of context and can miss surrounding logic (a conditional, a comment, a different " + + "code path) that changes what the match actually means; a full read does not have that limitation. " + + "Do not let a later, narrower result override an earlier, complete one just because it came later."; + +// One-time, no-tools self-check appended after the model's first draft final +// answer (see the verification-pass logic in the loop below). Targets a +// specific, observed failure (plan.md, 2026-08-27: "gave a verifiably wrong +// answer... appears to have trusted a later, narrower github_search_code +// result... over the complete file it had already read") that the +// SYSTEM_PREAMBLE rules above are meant to prevent DURING synthesis -- this +// is the backstop for when they don't: a forced second pass, after the +// draft answer already exists as concrete text to check claim-by-claim, +// rather than trusting the first synthesis attempt to have applied its own +// instructions correctly under a single pass. +const VERIFICATION_PROMPT = + "[SYSTEM NOTE -- verification pass, no tools available this turn] Before your answer above is " + + "treated as final, check it against the evidence you already gathered in this conversation. Go back " + + "through the RAW tool results already in this conversation -- not your own summary of them -- and " + + "confirm every specific factual claim in your answer (file paths, line numbers, function/variable " + + "names, log entries, statuses, dates, verdicts like 'consistent' or 'stale') is directly supported by " + + "something you actually retrieved. If a full/direct read of a file or page conflicts with a narrower " + + "or derived result (a search snippet, a grep match) that your answer relied on, the full/direct read " + + "is the more authoritative source -- prefer it and correct your answer accordingly. If you find " + + "anything unsupported or contradicted, fix it now. Respond with the corrected final answer (or the " + + "same answer, if it already holds up under this check) as plain text only -- you cannot call any " + + "functions this turn."; // Runs the investigation loop. Returns { answer, steps, transcript, runId, // failed? } where transcript is a human-readable log of each function call From edfb44c4f6e55e4edb6ae9fe5dfb83ce9ef4a040 Mon Sep 17 00:00:00 2001 From: allocsys <225476909+allocsys@users.noreply.github.com> Date: Thu, 27 Aug 2026 04:05:30 +0330 Subject: [PATCH 3/8] agent_delegate.js: declare and restore pendingVerification loop state --- connectors/gemini/agent_delegate.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/connectors/gemini/agent_delegate.js b/connectors/gemini/agent_delegate.js index 680eef1..23850c9 100644 --- a/connectors/gemini/agent_delegate.js +++ b/connectors/gemini/agent_delegate.js @@ -1026,6 +1026,15 @@ export async function runInvestigation({ task, max_steps = 20, resume_run_id, pr let repeatCounts = new Map(); let resultCache = new Map(); let consecutiveAllRepeatSteps = 0; + // Verification pass (2026-08-27, see VERIFICATION_PROMPT's comment above + // for the specific failure it targets): true once the model has produced + // a draft final answer and been sent back for one no-tools self-check + // round before that answer is trusted. Persisted across resumes (below) + // so a run that dies mid-verification -- e.g. the verification call + // itself hits a transient 429/503 -- resumes into the verification turn + // again rather than silently re-entering normal tool-use and re-drafting + // a whole new answer from scratch. + let pendingVerification = false; // How many entries of `contents` have already been pushed to the Redis // checkpoint list (fix #5) -- saveCheckpoint only ever needs the SLICE // added since the last checkpoint, not the whole array, so this cursor is @@ -1058,6 +1067,10 @@ export async function runInvestigation({ task, max_steps = 20, resume_run_id, pr // `checkpoint.task || task` below. repeatCounts = new Map(Object.entries(checkpoint.repeatCounts || {})); consecutiveAllRepeatSteps = checkpoint.consecutiveAllRepeatSteps || 0; + // Checkpoints saved before this field existed won't have it -- default + // to false (normal tool-use resumes as before), same defensive pattern + // as every other field restored here. + pendingVerification = checkpoint.pendingVerification || false; // Prefer the checkpoint's own record of the original task -- `task` is // genuinely ignored on a live resume (see file header), so this is the // only reliable source once a run is past step 1. Checkpoints saved From 46084ce5d809fb65b31186fb61cd8a7c20cd8320 Mon Sep 17 00:00:00 2001 From: allocsys <225476909+allocsys@users.noreply.github.com> Date: Thu, 27 Aug 2026 04:05:54 +0330 Subject: [PATCH 4/8] agent_delegate.js: withholdTools also covers the verification-pass turn --- connectors/gemini/agent_delegate.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/connectors/gemini/agent_delegate.js b/connectors/gemini/agent_delegate.js index 23850c9..6157ecc 100644 --- a/connectors/gemini/agent_delegate.js +++ b/connectors/gemini/agent_delegate.js @@ -1155,7 +1155,14 @@ export async function runInvestigation({ task, max_steps = 20, resume_run_id, pr // lesson as isFinalStep's own history, see its comment above), so this // reuses that structural fix instead of a new mechanism. const stuckLoopForce = consecutiveAllRepeatSteps >= 3; - const withholdTools = isFinalStep || stuckLoopForce; + // Verification pass (see VERIFICATION_PROMPT above): once a draft final + // answer has been sent back for self-checking, this turn must also be + // no-tools -- the point is to make the model re-examine evidence it + // already gathered, not go fetch more of it, and withholding tools is + // the same structural guarantee isFinalStep/stuckLoopForce already rely + // on (a text-only SYSTEM NOTE alone wasn't trusted for either of those, + // per their own history above -- no reason to trust it here instead). + const withholdTools = isFinalStep || stuckLoopForce || pendingVerification; let candidate; try { candidate = await providerChat(contents, { provider: effectiveProvider, tools: withholdTools ? undefined : FUNCTION_DECLARATIONS, model: effectiveModel, maxOutputTokens: effectiveMaxOutputTokens }); From 27f9acf471d0725b70631bde8c5f8e7931fa2471 Mon Sep 17 00:00:00 2001 From: allocsys <225476909+allocsys@users.noreply.github.com> Date: Thu, 27 Aug 2026 04:06:17 +0330 Subject: [PATCH 5/8] agent_delegate.js: trigger self-verification pass on a draft final answer with budget remaining --- connectors/gemini/agent_delegate.js | 58 +++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 12 deletions(-) diff --git a/connectors/gemini/agent_delegate.js b/connectors/gemini/agent_delegate.js index 6157ecc..158b64b 100644 --- a/connectors/gemini/agent_delegate.js +++ b/connectors/gemini/agent_delegate.js @@ -1209,23 +1209,57 @@ export async function runInvestigation({ task, max_steps = 20, resume_run_id, pr if (!functionCalls.length) { const answer = parts.map((p) => p.text || "").join("").trim(); + + // First arrival at a draft final answer, with tool access still + // available this turn (not already a forced no-tools turn) and steps + // left in the budget: don't trust it at face value yet -- push it + // back for one additional no-tools self-check turn first. See + // VERIFICATION_PROMPT's comment above for the specific, observed + // failure this targets (a later, narrower search result trusted over + // a complete file already read earlier in the same run). If no steps + // remain, or this turn was ALREADY a forced no-tools turn (final + // step, stuck-loop, or the verification pass itself -- all captured + // by withholdTools), fall through to the ordinary return below + // instead: there's no budget left to check, or this IS the check. + if (answer && !withholdTools && step < cappedSteps) { + contents.push({ role: "model", parts }); + contents.push({ role: "user", parts: [{ text: VERIFICATION_PROMPT }] }); + pendingVerification = true; + await saveCheckpoint(runId, { + newContents: contents.slice(contentsCheckpointedUpTo), + transcript, + stepsDone: step, + task: effectiveTask, + repeatCounts: Object.fromEntries(repeatCounts), + consecutiveAllRepeatSteps, + provider: effectiveProvider, + model: effectiveModel, + maxOutputTokens: effectiveMaxOutputTokens, + pendingVerification, + }); + contentsCheckpointedUpTo = contents.length; + continue; + } + await deleteCheckpoint(runId); if (!answer) { - // MALFORMED_FUNCTION_CALL on the final step specifically means: this - // step had NO tools in the request (isFinalStep withholds them - // entirely, see above), but the model tried to make a function call - // anyway -- Gemini rejects that as malformed rather than falling - // back to text. Observed concretely with max_steps: 1 on a task that - // genuinely needed a file read: the model had no way to answer - // without a tool, no tools were offered, and the result was this - // opaque finishReason with zero explanation of why (2026-07-26 - // stress test). Surface the actual cause instead of just the raw - // enum value, since "try a higher max_steps" is the fix and the - // caller has no way to infer that from "MALFORMED_FUNCTION_CALL" - // alone. + // MALFORMED_FUNCTION_CALL on a no-tools turn specifically means: + // this step had NO tools in the request (isFinalStep/stuckLoopForce/ + // pendingVerification all withhold them, see withholdTools above), + // but the model tried to make a function call anyway -- Gemini + // rejects that as malformed rather than falling back to text. + // Observed concretely with max_steps: 1 on a task that genuinely + // needed a file read: the model had no way to answer without a + // tool, no tools were offered, and the result was this opaque + // finishReason with zero explanation of why (2026-07-26 stress + // test). Surface the actual cause instead of just the raw enum + // value, since "try a higher max_steps" is the fix and the caller + // has no way to infer that from "MALFORMED_FUNCTION_CALL" alone. const starvationNote = withholdTools && candidate.finishReason === "MALFORMED_FUNCTION_CALL" ? (isFinalStep ? ` This was the final allowed step, which never includes tools (so the model can only answer in plain text here) -- but the model attempted a function call anyway, which Gemini rejects as malformed when no tools are available. This almost always means the task genuinely requires at least one tool call and max_steps (${cappedSteps}) left no tool-enabled steps to make it in. Retry with a higher max_steps (at least 2, ideally the default of 6 for anything non-trivial).` + : pendingVerification + ? ` This was the verification pass (no tools offered on purpose -- see VERIFICATION_PROMPT), but the model attempted a function call anyway, which Gemini rejects as malformed when no tools are available. The draft answer from the step before this one was never returned to the caller as a result -- treat this run as having produced no usable answer, and consider retrying with a higher max_steps in case the verification turn simply needed more room.` : ` This step had no tools available because ${consecutiveAllRepeatSteps} consecutive steps consisted entirely of repeat calls (same function + arguments already tried this run) -- fix #4's stuck-loop guard forces a text-only answer the same way the final step does, but the model attempted a function call anyway, which Gemini rejects as malformed when no tools are available. The task likely needs to be narrowed or rephrased so it doesn't require repeating the same information-gathering calls.`) : ""; return { answer: `(Gemini stopped without a final answer -- finishReason: ${candidate.finishReason || "unknown"})${starvationNote}`, steps: step, transcript, runId, task: effectiveTask }; From 4685b2f71aac4a894230dc7eaa89ec7182636b78 Mon Sep 17 00:00:00 2001 From: allocsys <225476909+allocsys@users.noreply.github.com> Date: Thu, 27 Aug 2026 04:06:33 +0330 Subject: [PATCH 6/8] agent_delegate.js: persist pendingVerification in the remaining saveCheckpoint call sites --- connectors/gemini/agent_delegate.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/connectors/gemini/agent_delegate.js b/connectors/gemini/agent_delegate.js index 158b64b..154beed 100644 --- a/connectors/gemini/agent_delegate.js +++ b/connectors/gemini/agent_delegate.js @@ -1185,6 +1185,7 @@ export async function runInvestigation({ task, max_steps = 20, resume_run_id, pr provider: effectiveProvider, model: effectiveModel, maxOutputTokens: effectiveMaxOutputTokens, + pendingVerification, }); const errMessage = err?.message ?? String(err); const redisOk = isRedisConfigured(); @@ -1421,6 +1422,7 @@ export async function runInvestigation({ task, max_steps = 20, resume_run_id, pr provider: effectiveProvider, model: effectiveModel, maxOutputTokens: effectiveMaxOutputTokens, + pendingVerification, }); const errMessage = err?.message ?? String(err); return { @@ -1481,6 +1483,14 @@ export async function runInvestigation({ task, max_steps = 20, resume_run_id, pr provider: effectiveProvider, model: effectiveModel, maxOutputTokens: effectiveMaxOutputTokens, + // Always false here in practice: this checkpoint fires only after a + // step that made function calls, and a verification-pass turn never + // reaches this branch (withholdTools forces it to text-only, so it + // either returns from the !functionCalls.length branch above or, on + // the MALFORMED_FUNCTION_CALL edge case, returns early with an + // error) -- included explicitly so the persisted checkpoint always + // states this field rather than silently omitting it on this path. + pendingVerification, }); contentsCheckpointedUpTo = contents.length; } From 27055d3666e4e28921ed0f60f445aca95c0e5153 Mon Sep 17 00:00:00 2001 From: allocsys <225476909+allocsys@users.noreply.github.com> Date: Thu, 27 Aug 2026 16:07:09 +0330 Subject: [PATCH 7/8] test: update agent-delegate-loop tests for mandatory verification pass --- test/agent-delegate-loop.test.js | 108 ++++++++++++++++++++++++++----- 1 file changed, 92 insertions(+), 16 deletions(-) diff --git a/test/agent-delegate-loop.test.js b/test/agent-delegate-loop.test.js index 77e33cf..2d07b55 100644 --- a/test/agent-delegate-loop.test.js +++ b/test/agent-delegate-loop.test.js @@ -13,6 +13,19 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; // environment without Redis provisioned -- no need to mock them, and this // also lets the "resume with no live checkpoint" error path be exercised // for real rather than through a mock. +// +// VERIFICATION PASS (2026-08-27, see plan.md "Gemini harness fix -- +// self-verification pass"): any draft final answer produced with tool +// budget still remaining (i.e. NOT already a forced no-tools turn, and at +// least one step left after it) now triggers one extra no-tools +// self-verification providerChat call before the answer is returned. This +// is provider-agnostic (lives in the loop body, not gemini-specific code), +// so every test below that reaches a draft answer with steps to spare needs +// a second (or third) mocked providerChat response for that verification +// round-trip, with step/call counts bumped by one accordingly. Tests that +// reach their draft answer on an already-withheld-tools turn (the final +// allowed step, or the stuck-loop force) are unaffected -- withholdTools +// being true is exactly what skips the verification pass. const mockProviderChat = vi.fn(); vi.mock("../connectors/llm/router.js", () => ({ @@ -46,28 +59,63 @@ describe.each(["gemini", "glm", "groq"])("agent_delegate.js — runInvestigation ({ runInvestigation } = await import("../connectors/gemini/agent_delegate.js")); }); - it("returns a plain-text answer on the first step with no function calls, threading the provider through", async () => { - mockProviderChat.mockResolvedValueOnce({ - content: { role: "model", parts: [{ text: "The answer is 42." }] }, - finishReason: "STOP", - }); + it("returns a plain-text answer after the mandatory verification pass, threading the provider through", async () => { + mockProviderChat + .mockResolvedValueOnce({ + content: { role: "model", parts: [{ text: "The answer is 42." }] }, + finishReason: "STOP", + }) + .mockResolvedValueOnce({ + content: { role: "model", parts: [{ text: "The answer is 42." }] }, + finishReason: "STOP", + }); const result = await runInvestigation({ task: "what is the answer", max_steps: 5, provider }); expect(result.answer).toBe("The answer is 42."); - expect(result.steps).toBe(1); + expect(result.steps).toBe(2); expect(result.failed).toBeUndefined(); - expect(mockProviderChat).toHaveBeenCalledTimes(1); + expect(mockProviderChat).toHaveBeenCalledTimes(2); const [, opts] = mockProviderChat.mock.calls[0]; expect(opts.provider).toBe(provider); + const [, verifyOpts] = mockProviderChat.mock.calls[1]; + expect(verifyOpts.provider).toBe(provider); + expect(verifyOpts.tools).toBeUndefined(); }); - it("executes a function call, feeds the result back, and returns the final answer on the next step", async () => { + it("runs a self-verification pass that can correct the draft answer before returning it", async () => { + mockProviderChat + .mockResolvedValueOnce({ + content: { role: "model", parts: [{ text: "The file is unused." }] }, + finishReason: "STOP", + }) + .mockResolvedValueOnce({ + content: { role: "model", parts: [{ text: "Correction: the file IS used, per the full file read." }] }, + finishReason: "STOP", + }); + + const result = await runInvestigation({ task: "check whether the file is used", max_steps: 5, provider }); + + expect(result.answer).toBe("Correction: the file IS used, per the full file read."); + expect(result.steps).toBe(2); + expect(mockProviderChat).toHaveBeenCalledTimes(2); + + const [verifyContents, verifyOpts] = mockProviderChat.mock.calls[1]; + expect(verifyOpts.tools).toBeUndefined(); + const lastMessage = verifyContents[verifyContents.length - 1]; + expect(lastMessage.parts[0].text).toMatch(/verification pass/i); + }); + + it("executes a function call, feeds the result back, runs the verification pass, and returns the final answer", async () => { mockProviderChat .mockResolvedValueOnce({ content: { role: "model", parts: [{ functionCall: { name: "github_get_repo_topics", args: { owner: "allocsys", repo: "madmcp" }, id: "call_1" } }] }, finishReason: "STOP", }) + .mockResolvedValueOnce({ + content: { role: "model", parts: [{ text: "Topics: mcp, ai" }] }, + finishReason: "STOP", + }) .mockResolvedValueOnce({ content: { role: "model", parts: [{ text: "Topics: mcp, ai" }] }, finishReason: "STOP", @@ -78,11 +126,15 @@ describe.each(["gemini", "glm", "groq"])("agent_delegate.js — runInvestigation const result = await runInvestigation({ task: "list topics", max_steps: 5, provider }); expect(result.answer).toBe("Topics: mcp, ai"); - expect(result.steps).toBe(2); - expect(mockProviderChat).toHaveBeenCalledTimes(2); - // Both steps used the same provider throughout a single run. + expect(result.steps).toBe(3); + expect(mockProviderChat).toHaveBeenCalledTimes(3); + // All three steps (call, draft answer, verification) used the same + // provider throughout a single run. expect(mockProviderChat.mock.calls[0][1].provider).toBe(provider); expect(mockProviderChat.mock.calls[1][1].provider).toBe(provider); + expect(mockProviderChat.mock.calls[2][1].provider).toBe(provider); + // The verification-pass call (3rd) had no tools available. + expect(mockProviderChat.mock.calls[2][1].tools).toBeUndefined(); // The transcript recorded the call, whatever its result (error or not). expect(result.transcript[0]).toMatch(/^\[step 1\] github_get_repo_topics/); }); @@ -100,6 +152,21 @@ describe.each(["gemini", "glm", "groq"])("agent_delegate.js — runInvestigation expect(opts.tools).toBeUndefined(); }); + it("skips the verification pass when it is itself the final allowed step", async () => { + mockProviderChat.mockResolvedValueOnce({ + content: { role: "model", parts: [{ text: "final answer, no budget to verify" }] }, + finishReason: "STOP", + }); + + const result = await runInvestigation({ task: "one step only, no room to verify", max_steps: 1, provider }); + + expect(result.answer).toBe("final answer, no budget to verify"); + expect(result.steps).toBe(1); + // No budget left after the draft answer arrives (it IS the final + // allowed step), so no second, verification-pass call is made. + expect(mockProviderChat).toHaveBeenCalledTimes(1); + }); + it("forces a text-only answer after 3 consecutive all-repeat steps (stuck-loop guard)", async () => { const repeatedCall = { content: { role: "model", parts: [{ functionCall: { name: "github_get_repo_topics", args: { owner: "a", repo: "b" }, id: "call_x" } }] }, @@ -115,9 +182,12 @@ describe.each(["gemini", "glm", "groq"])("agent_delegate.js — runInvestigation const result = await runInvestigation({ task: "keep repeating", max_steps: 10, provider }); expect(result.answer).toBe("giving up, here's what I found"); - // Step 5 (index 4) must have been called with tools withheld. + // Step 5 (index 4) must have been called with tools withheld -- and, + // because that turn was already a forced no-tools turn (the stuck-loop + // guard), it also skips the verification pass, so no 6th call is made. const step5Opts = mockProviderChat.mock.calls[4][1]; expect(step5Opts.tools).toBeUndefined(); + expect(mockProviderChat).toHaveBeenCalledTimes(5); }); it("stops after the hard step cap without a final answer", async () => { @@ -149,14 +219,20 @@ describe.each(["gemini", "glm", "groq"])("agent_delegate.js — runInvestigation }); it("falls through to a fresh run when resume_run_id's checkpoint is missing but a task is supplied", async () => { - mockProviderChat.mockResolvedValueOnce({ - content: { role: "model", parts: [{ text: "fresh run answer" }] }, - finishReason: "STOP", - }); + mockProviderChat + .mockResolvedValueOnce({ + content: { role: "model", parts: [{ text: "fresh run answer" }] }, + finishReason: "STOP", + }) + .mockResolvedValueOnce({ + content: { role: "model", parts: [{ text: "fresh run answer" }] }, + finishReason: "STOP", + }); const result = await runInvestigation({ task: "fallback task", resume_run_id: "nonexistent-run-id", max_steps: 5, provider }); expect(result.answer).toBe("fresh run answer"); + expect(mockProviderChat).toHaveBeenCalledTimes(2); // A fresh run gets its own new runId, not the (nonexistent) resume target. expect(result.runId).not.toBe("nonexistent-run-id"); }); From 3504bd4df6bf25f9e58c5a0fa4baed18d7f67cfb Mon Sep 17 00:00:00 2001 From: allocsys <225476909+allocsys@users.noreply.github.com> Date: Thu, 27 Aug 2026 17:07:05 +0330 Subject: [PATCH 8/8] plan.md: log live delegate_agent test of the validateFunctionArgs claim against main (pre-merge), and why it's inconclusive --- plan.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/plan.md b/plan.md index 9328bcd..777bda2 100644 --- a/plan.md +++ b/plan.md @@ -522,6 +522,29 @@ catches a similar contradiction or the original bug simply doesn't recur -- either result is useful signal, but treat this as unverified in production until that live run happens. +**Pre-merge live check (2026-08-27, against `main`, provider `gemini`, +no verification pass -- baseline only):** ran a live `delegate_agent` +task asking Gemini to directly determine, on `main`, whether +`validateFunctionArgs()` runs conditionally or unconditionally before +`execute()` in `agent_delegate.js` -- the same underlying fact the +original 13-step run got wrong. This run (14 steps) answered correctly: +1 `execute()` call site, `validateFunctionArgs()` unconditional before +it, code search agreeing with the full-file read. **Treat this as weak +signal, not a validation of anything:** the task was narrow and pointed +nearly directly at the fact in question, unlike the original open-ended +13-step run where the wrong claim emerged from synthesizing across a +longer, less targeted investigation -- not a faithful reproduction of the +failure conditions. LLM output is also non-deterministic, so one correct +run (on an easier task than the original) is not evidence the underlying +failure mode is gone. This run also did not exercise the verification +pass at all (ran against `main`, pre-merge). **Still needed before trusting +this branch in production:** re-run a task that faithfully reproduces the +original open-ended, multi-step conditions (not a narrowed version) -- +ideally several times against `main` first to establish an actual baseline +recurrence rate, then the same task the same number of times against this +branch post-merge, to get a real before/after comparison instead of a +single anecdote either way. + ## Designer notes (future phase-2 port, not this plan's scope) `connectors/frontend/designer_delegate.js` imports `geminiChat`/