main is red: StaleCheckpointFreshTurnTests fail after fe4ed3a5 landed on top of 1b45c641

docs/operating/agent-turn-checkpoints.md

@@ -99,14 +99,17 @@ accepts it only when the checkpoint already contains meaningful source changes
 relative to the pre-turn work-branch tip. An allow-empty checkpoint followed by
 a no-op resume uses the normal initial-work/rework no-diff failure path.
 
-When the work branch legitimately moves after the checkpoint was written — a
+When the base moves under the checkpoint after it was written — a
 pickup-time rebase onto a fresh base or a restart-recovery recomposition — the
-checkpoint commit no longer descends from the current pre-turn tip and is
-unusable, but that says nothing about the work. The turn preparation discards
-the stale checkpoint (logged as `Discarding durable agent-turn resume
+base ref changes across the pickup's upstream refresh and the checkpoint tree
+no longer applies, but that says nothing about the work. The turn preparation
+discards the stale checkpoint (logged as `Discarding durable agent-turn resume
 checkpoint`, the same record as a phase-boundary discard) and runs a normal
 fresh turn from the current branch tip with the branch preserved. This is not a
-failure and does not count toward terminal failures. A checkpoint backed by a
+failure and does not count toward terminal failures. A same-base advance of
+the work branch (an outside writer landing commits without moving the base)
+is not stale: the base ref is unchanged across the refresh, so the checkpoint
+still applies and resumes against the fork-point baseline. A checkpoint backed by a
 retained-sandbox recovery lease is never discarded this way: provider-owned
 state stays authoritative and the resume is refused explicitly.
 

src/CodeyBox.Orchestrator/PipelineRunner.AgentTurnResume.cs

@@ -1209,24 +1209,34 @@ private static async Task<bool> IsPreTurnCommitAncestorOfCheckpointAsync(
     }
 
     /// <summary>
-    /// True when the published checkpoint commit no longer descends from the
-    /// pre-turn work-branch tip, e.g. the branch was rebased onto a fresh base
-    /// or recomposed by restart recovery after the checkpoint was written. The
-    /// checkpoint tree is then unusable, but that says nothing about the work.
+    /// True when the published checkpoint is stale: the base-branch tip moved
+    /// between the pickup's upstream refresh and turn preparation, so the
+    /// base the checkpoint was captured against was superseded (a pickup-time
+    /// rebase onto a fresh base or a restart-recovery recomposition). The
+    /// resume path checks the checkpoint tree out as-is and reconciles onto
+    /// the branch tip, which cannot span a base move — so the checkpoint tree
+    /// is unusable, but that says nothing about the work and the caller
+    /// discards the bookkeeping for a fresh turn. The move is detected as a
+    /// pre/post-refresh tip difference (<paramref name="baseTipBeforeRefresh"/>
+    /// was snapshotted before the pickup refresh ran): a same-base branch
+    /// advance leaves the base untouched and is NOT stale — it keeps resuming
+    /// against the fork-point baseline downstream. Neither the branch tip nor
+    /// the fork point can prove staleness (a fork point is an ancestor of the
+    /// checkpoint by construction, and the branch tip diverges on every
+    /// ordinary advance). Both resolutions below are host-side against the
+    /// authoritative bare repo, never the possibly trailing sandbox clone.
     /// The checkpoint commit resolves from its content binding for typed refs,
-    /// or from the published ref for legacy preempt checkpoints. Both shapes
-    /// are canonical hex after this resolution, so the merge-base argv below
-    /// carries no untrusted flag characters.
+    /// or from the published ref for legacy preempt checkpoints; both shapes
+    /// are validated canonical hex before the ancestry probe.
     /// </summary>
-    private static async Task<bool> IsStaleAgentTurnCheckpointAsync(
-        ISandbox sandbox,
+    private async Task<bool> IsStaleAgentTurnCheckpointAsync(
+        string repoId,
+        string baseBranch,
         string preemptCheckpointRef,
-        string checkpointBranch,
-        string? preTurnCommitSha,
+        string? baseTipBeforeRefresh,
         CancellationToken ct)
     {
-        if (string.IsNullOrWhiteSpace(preTurnCommitSha))
-            throw new InvalidOperationException("Durable agent-turn resume has no pre-turn source commit.");
+        Validation.ValidateBranchName(baseBranch, nameof(baseBranch));
         string checkpointCommitSha;
         if (AgentTurnCheckpointRef.TryParse(preemptCheckpointRef, out var typedRef)
             && typedRef is not null)
@@ -1235,31 +1245,29 @@ private static async Task<bool> IsStaleAgentTurnCheckpointAsync(
         }
         else
         {
-            var resolved = await sandbox.ExecAsync(new SandboxExec
-            {
-                Argv =
-                [
-                    "git", "-C", SandboxConventions.WorkDir,
-                    "rev-parse", "--verify", $"origin/{checkpointBranch}",
-                ],
-                MaxStdoutBytes = 4096,
-                MaxStderrBytes = 4096,
-                KillOnOutputLimit = true,
-            }, ct);
-            PipelineAgentExecutor.ThrowIfExecutionUnavailable(resolved);
-            if (!resolved.Success)
-            {
-                throw new InvalidOperationException(
-                    "Failed to resolve the durable agent-turn checkpoint ref.");
-            }
-            checkpointCommitSha = resolved.Stdout.Trim();
+            checkpointCommitSha = await _gitHost.ResolveCommitAsync(
+                repoId, preemptCheckpointRef, ct).ConfigureAwait(false);
         }
-
-        return !await IsPreTurnCommitAncestorOfCheckpointAsync(
-            sandbox,
-            preTurnCommitSha,
-            checkpointCommitSha,
-            ct).ConfigureAwait(false);
+        Validation.ValidateCommitSha(checkpointCommitSha, nameof(checkpointCommitSha));
+        var baseTip = await _gitHost.ResolveCommitAsync(
+            repoId, $"refs/heads/{baseBranch}", ct).ConfigureAwait(false);
+        Validation.ValidateCommitSha(baseTip, nameof(baseTip));
+        if (string.IsNullOrWhiteSpace(baseTipBeforeRefresh))
+        {
+            // No pre-refresh snapshot (the repo or base was absent at pickup):
+            // fall back to base ancestry — a checkpoint that does not descend
+            // from the current base cannot resume.
+            return !await _gitHost.IsAncestorAsync(
+                repoId,
+                baseTip.ToLowerInvariant(),
+                checkpointCommitSha.ToLowerInvariant(),
+                ct).ConfigureAwait(false);
+        }
+        Validation.ValidateCommitSha(baseTipBeforeRefresh, nameof(baseTipBeforeRefresh));
+        return !string.Equals(
+            baseTipBeforeRefresh.ToLowerInvariant(),
+            baseTip.ToLowerInvariant(),
+            StringComparison.Ordinal);
     }
 
     private const string StaleCheckpointLeaseRefusalMessage =

src/CodeyBox.Orchestrator/PipelineRunner.Rebase.cs

@@ -1250,7 +1250,7 @@ internal sealed record WorkBranchStagingProof(string HostTip, string SandboxHead
     /// never throws for a missing branch.
     /// </summary>
     private async Task<string?> TryResolveHostBranchTipAsync(
-        string repoId, string branch, CancellationToken ct)
+        string repoId, string branch, CancellationToken ct, string scope = "work branch")
     {
         try
         {
@@ -1261,8 +1261,8 @@ internal sealed record WorkBranchStagingProof(string HostTip, string SandboxHead
         catch (Exception ex) when (ex is not OperationCanceledException)
         {
             _log.LogDebug(ex,
-                "Could not resolve host tip of work branch '{Branch}' for staging log; continuing without it",
-                branch);
+                "Could not resolve host tip of {Scope} '{Branch}' for staging log; continuing without it",
+                scope, branch);
             return null;
         }
     }
@@ -1278,6 +1278,11 @@ internal sealed record WorkBranchStagingProof(string HostTip, string SandboxHead
     /// ancestor, which keeps the validation in the safe direction. Falls back
     /// to the raw tip when the checkpoint source is unknown (legacy refs) or
     /// the fork point cannot be computed.
+    /// This baseline is only for the resumed no-op/meaningful-changes
+    /// validation. Whether the checkpoint is usable at all (stale discard) is
+    /// decided separately against the current base-branch tip — a fork point
+    /// is an ancestor of the checkpoint by construction and can never detect
+    /// a base move.
     /// </summary>
     private async Task<string> ResolveAgentTurnResumeBaseAsync(
         string repoId,

src/CodeyBox.Orchestrator/PipelineRunner.WorkPhase.cs

@@ -52,7 +52,12 @@ public sealed partial class PipelineRunner
         // timings, streams, supervision, and prompt preprocessing instead of
         // "rework". Null keeps the legacy isInitial-derived labels.
         string? phaseLabelOverride = null,
-        AgentPromptPhase? promptPhaseOverride = null)
+        AgentPromptPhase? promptPhaseOverride = null,
+        // Base-branch tip snapshotted before the pickup's upstream refresh.
+        // Compared against the base tip at turn-preparation time: any move
+        // means the checkpoint's base was superseded and the checkpoint is
+        // stale. Null skips the comparison (fresh pickup without a resume).
+        string? preRefreshBaseTipSha = null)
     {
         item = await RefreshAgentTurnResumeCheckpointAsync(item, isInitial, iteration, ct);
         var resumingGitCheckpoint = !string.IsNullOrWhiteSpace(item.PreemptCheckpoint);
@@ -320,20 +325,23 @@ await ConvertRetainedSandboxToCheckpointAsync(
                 checkpointBranch = ValidatePreemptCheckpoint(item, preemptCheckpoint);
                 await PipelineAgentExecutor.Run(sandbox, "git", "-C", SandboxConventions.WorkDir, "fetch", "origin", preemptCheckpoint);
                 if (await IsStaleAgentTurnCheckpointAsync(
-                        sandbox, preemptCheckpoint, checkpointBranch, resumePreTurnCommitSha, ct).ConfigureAwait(false))
+                        repoId, baseBranch, preemptCheckpoint, preRefreshBaseTipSha, ct).ConfigureAwait(false))
                 {
-                    // The work branch legitimately moved after the checkpoint
-                    // was written (a pickup-time rebase onto a fresh base or a
-                    // restart-recovery recomposition). The checkpoint tree is
+                    // The base moved under the checkpoint after it was
+                    // written (a pickup-time rebase onto a fresh base or a
+                    // restart-recovery recomposition moved the base ref across
+                    // this pickup's upstream refresh). The checkpoint tree is
                     // unusable, but that says nothing about the work: discard
                     // the bookkeeping and run a normal fresh turn from the
                     // current branch tip, preserving the branch — never a
-                    // terminal failure. A retained-sandbox lease still refuses
-                    // inside the discard: provider-owned state is never
-                    // silently dropped.
+                    // terminal failure. A same-base branch advance leaves the
+                    // base untouched, so it still resumes below against the
+                    // fork-point baseline. A retained-sandbox lease still
+                    // refuses inside the discard: provider-owned state is
+                    // never silently dropped.
                     item = await DiscardStaleAgentTurnCheckpointAsync(
                         item,
-                        "durable agent-turn checkpoint no longer descends from its pre-turn work-branch commit",
+                        "durable agent-turn checkpoint base moved across the pickup upstream refresh",
                         runner.Kind,
                         agentPhase,
                         ct).ConfigureAwait(false);

src/CodeyBox.Orchestrator/PipelineRunner.cs

@@ -316,6 +316,18 @@ await TransitionFailed(item,
         {
             await using var sandboxContext = new WorkSandboxContext(_sandboxes, _pipelineTuning, _log);
             var configuredBaseBranch = item.BaseBranch ?? project.DefaultBaseBranch;
+            // Snapshot the base-branch tip BEFORE the ensure below can
+            // refresh it from upstream. A durable agent-turn checkpoint is
+            // captured against the base it staged on: when the refresh moves
+            // the base, the checkpoint tree no longer applies and the work
+            // phase must discard it for a fresh turn. The branch tip alone
+            // cannot prove this — a same-base branch advance must keep
+            // resuming — so the pre/post-refresh comparison is the stale
+            // signal. Best-effort: a missing repo/branch yields null and the
+            // work phase falls back to base ancestry.
+            var baseTipBeforeRefresh = resumingPreempt && !string.IsNullOrWhiteSpace(configuredBaseBranch)
+                ? await TryResolveHostBranchTipAsync(item.Id.ToString(), configuredBaseBranch, ct, scope: "base branch")
+                : null;
             var (repoId, baseBranch) = await EnsurePipelineRepositoryAsync(item, project, configuredBaseBranch, ct);
             var hadRecordedWorkBranchAtEntry = !string.IsNullOrWhiteSpace(item.WorkBranch);
             var workBranch = item.WorkBranch ?? DefaultWorkBranchFor(item.Id);
@@ -590,7 +602,8 @@ await RunWithStuckProbeAsync(trialItem, project, runner.Kind, "work", workPhase,
                                         hostShutdownToken,
                                         buildFailurePolicy: RequiredBuildPolicy.Terminal,
                                         auditorsForPreemptiveSelfReview: auditors,
-                                        resumePreTurnCommitSha: resumePreTurnCommitSha),
+                                        resumePreTurnCommitSha: resumePreTurnCommitSha,
+                                        preRefreshBaseTipSha: baseTipBeforeRefresh),
                                     workToken: attemptCt),
                             ct,
                             phaseCancellation: workPhase,
@@ -658,7 +671,8 @@ trialItem.PreemptCheckpoint is { } checkpointRef
                                         // build gate and folded into the iteration's findings.
                                         buildFailurePolicy: RequiredBuildPolicy.DeferToAuditLoop,
                                         iteration: resumeIteration,
-                                        resumePreTurnCommitSha: resumePreTurnCommitSha),
+                                        resumePreTurnCommitSha: resumePreTurnCommitSha,
+                                        preRefreshBaseTipSha: baseTipBeforeRefresh),
                                     workToken: attemptCt),
                             ct,
                             phaseCancellation: reworkPhase,