Merge-result build verification has its own 15-min timeout, shorter than the required-build budget it runs: every merge is cancelled

docs/reference/configuration.md

@@ -580,7 +580,9 @@ Hot-reloadable retry and recovery bounds used by pipeline execution.
 | `SuggestionDedupeSimilarityThreshold` | `0.6` | Minimum Jaccard similarity over a suggestion's normalized token signature (lower-cased, punctuation- and stop-word-stripped title tokens plus normalized file paths) for a repeat to merge into an existing row instead of creating a duplicate. Must be in (0, 1]. Hot-reloadable. |
 | `MergeLandingMaxAttempts` | `3` | Times a merge phase re-queues its landing after the base branch moves between composition and the conditional base-ref update. Retries do not consume the guard-rework budget. Past the cap the item parks at `MergeConflictResolutionFailed` with the recorded reason. Hot-reloadable. |
 | `MergeResultBuildVerificationEnabled` | `true` | Builds the exact merge-result tree with the required build in a sandbox before advancing the base branch, inside the shared per-base-branch landing queue. A failed build is a semantic conflict: the tree is not landed and the item takes conflict-rework turns briefed with the build errors. Hot-reloadable. |
-| `MergeResultBuildVerificationTimeout` | `00:15:00` (15 min) | Upper bound for a single merge-result build verification. Exceeding it fails the verification (the item re-queues through rework and re-verifies) — an unverified tree is never landed. Must be positive. Hot-reloadable. |
+| `MergeResultBuildVerificationTimeout` | unset (automatic) | Upper bound for a single merge-result build verification. Unset means the required-build budget plus `MergeResultBuildVerificationProvisioningAllowance`. An explicit value below that floor is clamped up (with a config-load warning), never applied as a silent cap. Exceeding the effective budget is infrastructure — retried up to `MergeResultBuildVerificationInfraMaxAttempts` without consuming the guard-rework budget — an unverified tree is never landed. Must be positive when set. Hot-reloadable. |
+| `MergeResultBuildVerificationProvisioningAllowance` | `00:05:00` (5 min) | Sandbox-provisioning allowance added to the required-build budget to derive the effective merge-result verification budget (the inner required-build timeout covers clone/checkout/build after the verifier sandbox exists; the merge-result timeout is measured from sandbox acquisition). Must be non-negative. Hot-reloadable. |
+| `MergeResultBuildVerificationInfraMaxAttempts` | `3` | Bounded retries for merge-result verifications that time out. Timeouts never consume the `MergeGuardReworkMaxAttempts` guard-rework budget; past this cap the item parks at `MergeConflictResolutionFailed` with a message naming both timeout settings. Must be ≥ 1. Hot-reloadable. |
 | `MergeGuardReworkMaxAttempts` | `2` | Conflict-rework turns after a resolver safety guard fires (edits outside the permitted conflict hunks, or a rework that discarded prior commits). Each turn runs on the refreshed base with the guard reason in the brief; past the cap the item parks at `MergeConflictResolutionFailed`. Hot-reloadable. |
 | `SuggestionDedupeDismissedMatchWindow` | `30.00:00:00` (30 days) | How far back a dismissed suggestion still attracts repeats: a matching suggestion dismissed within this window bumps its occurrence count instead of spawning a new row (the dismissal sticks). Open suggestions always match. Hot-reloadable. |
 | `SuggestionDedupeMaxRecordedSources` | `25` | Cap on the source work-item id list recorded on a merged suggestion. Once full, repeats still bump the occurrence count but stop appending ids. Must be ≥ 1. Hot-reloadable. |

src/CodeyBox.Api/AgentConfigHotReload.cs

@@ -272,6 +272,7 @@ public async Task StartAsync(CancellationToken cancellationToken)
         await ApplyConfiguredAgentPausesAtStartupAsync(initial, cancellationToken);
 
         _subscription = _monitor.OnChange(OnConfigChanged);
+        LogMergeResultBuildBudgetWarningIfTooTight(initial);
         _log.LogInformation(
             "AgentConfigHotReload subscribed to CodeyBoxOptions: classes={ClassesLen} concurrency={ConcurrencyLen} burn={BurnLen} pricing={PricingLen} defaults={DefaultsLen} sanitizer={SanitizerLen} tolerance={ToleranceLen}",
             _lastRouter.Length, _lastConcurrency.Length, _lastBurn.Length, _lastPricing.Length, _lastDefaults.Length, _lastSanitizer.Length, _lastNetworkTolerance.Length);
@@ -1508,7 +1509,13 @@ private void ApplyPipelineTuningIfChanged(CodeyBoxOptions opts)
 
         var next = SerializePipelineTuning(opts.PipelineTuning);
         if (string.Equals(_lastPipelineTuning, next, StringComparison.Ordinal))
+        {
+            // The merge-result floor also depends on the required-build budget
+            // (restart-required): re-check the clamp warning even when the
+            // tuning block itself did not change.
+            LogMergeResultBuildBudgetWarningIfTooTight(opts);
             return;
+        }
 
         var prev = _lastPipelineTuning;
         try
@@ -1519,6 +1526,7 @@ private void ApplyPipelineTuningIfChanged(CodeyBoxOptions opts)
             _lastPipelineTuning = next;
             AuditLog.ConfigReloaded("PipelineTuning", prev, next);
             _log.LogInformation("Hot-reloaded PipelineTuning: {OldValue} → {NewValue}", prev, next);
+            LogMergeResultBuildBudgetWarningIfTooTight(opts);
         }
         catch (Exception ex)
         {
@@ -1529,6 +1537,22 @@ private void ApplyPipelineTuningIfChanged(CodeyBoxOptions opts)
         }
     }
 
+    /// <summary>
+    /// Config-load validation warning for the merge-result build budget: the
+    /// effective per-attempt budget is never below the required-build budget
+    /// plus the provisioning allowance, so an explicit
+    /// <c>MergeResultBuildVerificationTimeout</c> below that floor is clamped
+    /// up at verification time. This warning makes the clamp visible at load
+    /// (and on every reload) instead of a silent cap.
+    /// </summary>
+    private void LogMergeResultBuildBudgetWarningIfTooTight(CodeyBoxOptions opts)
+    {
+        ArgumentNullException.ThrowIfNull(opts);
+        var required = TimeSpan.FromSeconds(Math.Max(60, opts.RequiredBuildVerificationTimeoutSeconds));
+        if (opts.PipelineTuning.TryGetMergeResultBuildBudgetWarning(required, out var warning))
+            _log.LogWarning("{Warning}", warning);
+    }
+
     private void ApplyBudgetDeferralRecheckIfChanged(CodeyBoxOptions opts)
     {
         if (_budgetDeferralRecheck is null) return;
@@ -1627,7 +1651,9 @@ internal static string SerializePipelineTuning(PipelineTuningOptions opts) =>
                 opts.MergeLandingMaxAttempts,
                 opts.MergeGuardReworkMaxAttempts,
                 opts.MergeResultBuildVerificationEnabled,
-                MergeResultBuildVerificationTimeoutSeconds = opts.MergeResultBuildVerificationTimeout.TotalSeconds,
+                MergeResultBuildVerificationTimeoutSeconds = opts.MergeResultBuildVerificationTimeout?.TotalSeconds,
+                MergeResultBuildVerificationProvisioningAllowanceSeconds = opts.MergeResultBuildVerificationProvisioningAllowance.TotalSeconds,
+                opts.MergeResultBuildVerificationInfraMaxAttempts,
                 opts.EnableSandboxReuse,
                 opts.MaxSandboxReuses,
                 MaxSandboxLifetimeSeconds = opts.MaxSandboxLifetime.TotalSeconds,

src/CodeyBox.Orchestrator/MergeScopeFence.cs

@@ -61,6 +61,24 @@ public MergeResultBuildFailedException(string message, string buildOutput, Excep
     }
 }
 
+/// <summary>
+/// A merge-result build verification that timed out without proving the
+/// merged tree: infrastructure, never a semantic verdict on the diff. The
+/// merge landing retries these up to
+/// <see cref="PipelineTuningOptions.MergeResultBuildVerificationInfraMaxAttempts"/>
+/// without consuming the <see cref="PipelineTuningOptions.MergeGuardReworkMaxAttempts"/>
+/// guard-rework budget (which exists for real semantic failures); only after
+/// the infra budget is spent does the item park, with a message naming both
+/// timeout settings so a too-tight budget is actionable.
+/// </summary>
+public sealed class MergeResultBuildVerificationTimeoutException : MergeConflictResolutionFailedException
+{
+    public MergeResultBuildVerificationTimeoutException(string message, Exception? innerException = null)
+        : base(message, innerException, failureKind: WorkItemFailureKinds.Infrastructure, phase: "merge")
+    {
+    }
+}
+
 /// <summary>
 /// Retryable signal that the base branch moved between merge composition
 /// and landing (the conditional <c>update-ref</c> refused to advance).
@@ -113,10 +131,15 @@ internal static bool IsResolverGuardFailure(MergeConflictResolutionFailedExcepti
         {
             if (current is MergeBaseMovedException)
                 return false;
+            if (current is MergeResultBuildVerificationTimeoutException)
+                return false;
             if (current is ScopeFenceViolation)
                 return true;
         }
 
+        if (ex is MergeResultBuildVerificationTimeoutException)
+            return false;
+
         return ex.Message.Contains("outside the permitted conflict hunks", StringComparison.Ordinal)
             || ex.Message.Contains("discarded prior commits", StringComparison.Ordinal)
             || ex.Message.Contains("was not part of the resolved diff", StringComparison.Ordinal)

src/CodeyBox.Orchestrator/PipelineRunner.Merge.cs

@@ -714,6 +714,20 @@ await _store.UpdateAsync(current with
     /// base. Verifier infrastructure failures propagate untouched (parked
     /// as infrastructure, never landed, never misattributed to the diff).
     /// </para>
+    ///
+    /// <para>
+    /// A verification that times out is infrastructure, not a verdict on the
+    /// diff: it is retried up to
+    /// <see cref="PipelineTuningOptions.MergeResultBuildVerificationInfraMaxAttempts"/>
+    /// without consuming the guard-rework budget, and only parks (as
+    /// <see cref="MergeResultBuildVerificationTimeoutException"/>) once the
+    /// infra budget is spent. The per-attempt budget is the single source of
+    /// truth in
+    /// <see cref="PipelineTuningOptions.GetEffectiveMergeResultBuildVerificationTimeout"/>,
+    /// never below the required-build budget plus the provisioning
+    /// allowance. Every attempt reports its duration in the structured log
+    /// so a too-tight budget is visible.
+    /// </para>
     /// </summary>
     private async Task VerifyMergeBuildAndLandAsync(
         WorkItem item,
@@ -753,28 +767,95 @@ private async Task VerifyMergeBuildAndLandAsync(
             await _gitHost.SetBranchToCommitAsync(repoId, verifyBranch, mergeSha, ct);
             try
             {
-                var timeout = _pipelineTuning.Current.MergeResultBuildVerificationTimeout;
-                using var timeoutCts = new CancellationTokenSource(timeout);
-                using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(ct, timeoutCts.Token);
-                RequiredBuildVerificationResult result;
-                try
+                // Snapshot the budgets once: an in-flight verification keeps a
+                // stable per-attempt budget while the next landing observes a
+                // configuration edit.
+                var tuning = _pipelineTuning.Current;
+                var requiredBuildTimeout = _opts.RequiredBuildVerificationTimeout;
+                var configuredMergeTimeout = tuning.MergeResultBuildVerificationTimeout;
+                var effectiveTimeout = tuning.GetEffectiveMergeResultBuildVerificationTimeout(requiredBuildTimeout);
+                var maxAttempts = Math.Max(1, tuning.MergeResultBuildVerificationInfraMaxAttempts);
+                if (tuning.TryGetMergeResultBuildBudgetWarning(requiredBuildTimeout, out var budgetWarning))
                 {
-                    result = await _requiredBuildGate.VerifyMergeResultAsync(
-                        item, project, repoId, baseBranch, verifyBranch, linkedCts.Token);
-                }
-                catch (OperationCanceledException ex) when (!ct.IsCancellationRequested && timeoutCts.IsCancellationRequested)
-                {
-                    throw new MergeConflictResolutionFailedException(
-                        $"merge-result build verification timed out after {timeout} without proving the merged tree; refusing to land an unverified tree",
-                        ex);
+                    _log.LogWarning(
+                        "Merge-result build verification budget clamped for work item {WorkItemId}: {Warning} " +
+                        "(configured MergeResultBuildVerificationTimeout={Configured}, " +
+                        "RequiredBuildVerificationTimeout={Required}, effective={Effective})",
+                        item.Id, budgetWarning, configuredMergeTimeout, requiredBuildTimeout, effectiveTimeout);
                 }
 
-                if (result.Status == RequiredBuildVerificationStatus.Failed)
+                for (var attempt = 1; ; attempt++)
                 {
-                    var summary = RequiredBuildGate.BuildFailureSummary(result);
-                    throw new MergeResultBuildFailedException(
-                        $"merge result failed required build (semantic conflict against '{baseBranch}'): {summary}",
-                        result.Output);
+                    var attemptStart = _opts.TimeProvider.GetUtcNow();
+                    // TimeProvider-backed so tests drive the budget with a
+                    // FakeTimeProvider instead of racing wall-clock scheduling.
+                    using var timeoutCts = new CancellationTokenSource(effectiveTimeout, _opts.TimeProvider);
+                    using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(ct, timeoutCts.Token);
+                    RequiredBuildVerificationResult result;
+                    try
+                    {
+                        result = await _requiredBuildGate.VerifyMergeResultAsync(
+                            item, project, repoId, baseBranch, verifyBranch, linkedCts.Token);
+                    }
+                    catch (OperationCanceledException ex) when (!ct.IsCancellationRequested && timeoutCts.IsCancellationRequested)
+                    {
+                        var timedOutElapsed = _opts.TimeProvider.GetUtcNow() - attemptStart;
+                        var retryAction = attempt < maxAttempts
+                            ? "retrying as infrastructure without consuming the guard-rework budget"
+                            : "infra retry budget spent; parking without landing the unverified tree";
+                        _log.LogWarning(
+                            ex,
+                            "Merge-result build verification attempt {Attempt}/{MaxAttempts} for work item {WorkItemId} " +
+                            "timed out after {Elapsed} (budget {Budget}); {Action}",
+                            attempt, maxAttempts, item.Id, timedOutElapsed, effectiveTimeout, retryAction);
+                        if (attempt < maxAttempts)
+                            continue;
+
+                        throw new MergeResultBuildVerificationTimeoutException(
+                            FormatMergeVerificationTimeoutMessage(
+                                maxAttempts, effectiveTimeout, timedOutElapsed,
+                                requiredBuildTimeout, configuredMergeTimeout),
+                            ex);
+                    }
+
+                    // The inner verifier's own required-build budget firing is
+                    // the same infrastructure outcome as the outer budget
+                    // firing: the build never proved the tree, so this is not
+                    // a semantic conflict and must not route to rework.
+                    if (SandboxRequiredBuildVerifier.IsBuildTimeoutResult(result))
+                    {
+                        var innerElapsed = _opts.TimeProvider.GetUtcNow() - attemptStart;
+                        var innerRetryAction = attempt < maxAttempts
+                            ? "retrying as infrastructure without consuming the guard-rework budget"
+                            : "infra retry budget spent; parking without landing the unverified tree";
+                        _log.LogWarning(
+                            "Merge-result build verification attempt {Attempt}/{MaxAttempts} for work item {WorkItemId} " +
+                            "hit the inner required-build timeout after {Elapsed} (budget {Budget}); {Action}",
+                            attempt, maxAttempts, item.Id, innerElapsed, effectiveTimeout, innerRetryAction);
+                        if (attempt < maxAttempts)
+                            continue;
+
+                        throw new MergeResultBuildVerificationTimeoutException(
+                            FormatMergeVerificationTimeoutMessage(
+                                maxAttempts, effectiveTimeout, innerElapsed,
+                                requiredBuildTimeout, configuredMergeTimeout));
+                    }
+
+                    var elapsed = _opts.TimeProvider.GetUtcNow() - attemptStart;
+                    _log.LogInformation(
+                        "Merge-result build verification attempt {Attempt}/{MaxAttempts} for work item {WorkItemId} " +
+                        "finished in {Elapsed} with status {Status} (budget {Budget})",
+                        attempt, maxAttempts, item.Id, elapsed, result.Status, effectiveTimeout);
+
+                    if (result.Status == RequiredBuildVerificationStatus.Failed)
+                    {
+                        var summary = RequiredBuildGate.BuildFailureSummary(result);
+                        throw new MergeResultBuildFailedException(
+                            $"merge result failed required build (semantic conflict against '{baseBranch}'): {summary}",
+                            result.Output);
+                    }
+
+                    break;
                 }
             }
             finally
@@ -790,6 +871,20 @@ private async Task VerifyMergeBuildAndLandAsync(
         }
     }
 
+    private static string FormatMergeVerificationTimeoutMessage(
+        int maxAttempts,
+        TimeSpan effectiveTimeout,
+        TimeSpan lastAttemptElapsed,
+        TimeSpan requiredBuildTimeout,
+        TimeSpan? configuredMergeTimeout)
+        => "merge-result build verification timed out after "
+            + $"{maxAttempts} attempt(s) of {effectiveTimeout} (last attempt ran {lastAttemptElapsed}) "
+            + "without proving the merged tree; refusing to land an unverified tree "
+            + $"(RequiredBuildVerificationTimeout={requiredBuildTimeout}, "
+            + $"PipelineTuning:MergeResultBuildVerificationTimeout={configuredMergeTimeout?.ToString() ?? "unset (automatic)"}). "
+            + "Raise MergeResultBuildVerificationTimeout (it must cover RequiredBuildVerificationTimeout "
+            + "plus sandbox provisioning) and re-dispatch the item.";
+
     /// <summary>
     /// Best-effort removal of the short-lived merge-verification branch.
     /// Deletes only when the ref still points at <paramref name="mergeSha"/>: