feat(engine): add graceful exhaustion and deterministic replay cache

README.md

@@ -297,6 +297,8 @@ Features adopted from open-source agent projects. All are off by default unless
 | Relevance pruning | `internal/relevanceprune` | Token-budgeted context pruning preserving recent turns/tool calls/errors. Wired into compaction as a `relevance` strategy |
 | Tool-result clearing | `internal/engine` (`ClearOldToolResults`) | Two-tier context management: at 80% of the context window, stale tool-result content is replaced with `[output cleared]` placeholders (tool_use kept intact) before compacting — a gentler tier below compaction |
 | Approval pause timing | `internal/permissions` | Approval requests record decision timestamp + human deliberation duration (`DecisionAt`/`PauseDuration`) for approval-latency observability |
+| Graceful exhaustion | `internal/engine` (`SynthesisForExhaustion`) | When turn/token/time limits hit, one final tools-disabled LLM call synthesizes a coherent completion (accomplished/remaining/next steps) instead of a bare stop line. Opt-in via `HAWK_GRACEFUL_EXHAUSTION=1` |
+| Deterministic replay cache | `internal/replaycache` (`HAWK_REPLAY_CACHE_DIR`) | Disk-persisted SHA-256-keyed cache of completions; identical requests replay stored responses for reproducible regression runs |
 
 ## Usage
 

internal/engine/chat_replay.go

@@ -0,0 +1,45 @@
+package engine
+
+import (
+	"context"
+	"os"
+
+	"github.com/GrayCodeAI/hawk/internal/replaycache"
+	"github.com/GrayCodeAI/hawk/internal/types"
+)
+
+// replayCacheDirEnv opts a run into the disk-persisted replay cache: when set,
+// every non-streaming completion is looked up by its canonicalized request and
+// replayed from disk on a hit, giving deterministic, offline regression runs.
+// Unset (the default) leaves the chat path untouched.
+const replayCacheDirEnv = "HAWK_REPLAY_CACHE_DIR"
+
+// replayFingerprintEnv optionally folds an extra string (e.g. a fixture
+// version) into replay cache keys so whole suites can be invalidated at once.
+const replayFingerprintEnv = "HAWK_REPLAY_FINGERPRINT"
+
+// replayKey builds the cache key for one completion request.
+func replayKey(opts types.ChatOptions, messages []types.EyrieMessage) string {
+	return replaycache.Key(replaycache.Fingerprint(os.Getenv(replayFingerprintEnv)),
+		opts.Provider, opts.Model, messages, opts.MaxTokens)
+}
+
+// chatWithReplay wraps client.Chat with the replay cache when
+// HAWK_REPLAY_CACHE_DIR is set; otherwise it calls straight through.
+func chatWithReplay(ctx context.Context, client ChatClient, messages []types.EyrieMessage, opts types.ChatOptions) (*types.EyrieResponse, error) {
+	dir := os.Getenv(replayCacheDirEnv)
+	if dir == "" {
+		return client.Chat(ctx, messages, opts)
+	}
+	cache := replaycache.New(dir)
+	key := replayKey(opts, messages)
+	if resp, ok := cache.Get(key); ok {
+		return resp, nil
+	}
+	resp, err := client.Chat(ctx, messages, opts)
+	if err != nil {
+		return resp, err
+	}
+	_ = cache.Put(key, resp) // best-effort: a failed write must not fail the turn
+	return resp, nil
+}

internal/engine/chat_replay_test.go

@@ -0,0 +1,118 @@
+package engine
+
+import (
+	"context"
+	"errors"
+	"path/filepath"
+	"testing"
+
+	"github.com/GrayCodeAI/hawk/internal/types"
+)
+
+func TestChatWithReplayDisabledPassesThrough(t *testing.T) {
+	t.Setenv(replayCacheDirEnv, "")
+	calls := 0
+	client := &countingClient{n: &calls, resp: "live"}
+	resp, err := chatWithReplay(context.Background(), client,
+		[]types.EyrieMessage{{Role: "user", Content: "hi"}}, types.ChatOptions{Model: "m"})
+	if err != nil || resp.Content != "live" {
+		t.Fatalf("resp=%v err=%v", resp, err)
+	}
+	if calls != 1 {
+		t.Fatalf("calls = %d, want 1", calls)
+	}
+}
+
+func TestChatWithReplayCachesAndHits(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv(replayCacheDirEnv, dir)
+
+	calls := 0
+	client := &countingClient{n: &calls, resp: "live"}
+	msgs := []types.EyrieMessage{{Role: "user", Content: "deterministic"}}
+	opts := types.ChatOptions{Provider: "p", Model: "m", MaxTokens: 10}
+
+	first, err := chatWithReplay(context.Background(), client, msgs, opts)
+	if err != nil || first.Content != "live" {
+		t.Fatalf("first call: resp=%v err=%v", first, err)
+	}
+	if calls != 1 {
+		t.Fatalf("calls after first = %d", calls)
+	}
+
+	// Second identical request must replay from disk without calling the client.
+	second, err := chatWithReplay(context.Background(), client, msgs, opts)
+	if err != nil || second.Content != "live" {
+		t.Fatalf("replayed call: resp=%v err=%v", second, err)
+	}
+	if calls != 1 {
+		t.Fatalf("client called %d times, want 1 (second served from cache)", calls)
+	}
+}
+
+func TestChatWithReplayFingerprintInvalidates(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv(replayCacheDirEnv, dir)
+	t.Setenv(replayFingerprintEnv, "v1")
+
+	calls := 0
+	client := &countingClient{n: &calls, resp: "live"}
+	msgs := []types.EyrieMessage{{Role: "user", Content: "x"}}
+	opts := types.ChatOptions{Provider: "p", Model: "m"}
+
+	if _, err := chatWithReplay(context.Background(), client, msgs, opts); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv(replayFingerprintEnv, "v2")
+	if _, err := chatWithReplay(context.Background(), client, msgs, opts); err != nil {
+		t.Fatal(err)
+	}
+	if calls != 2 {
+		t.Fatalf("calls = %d, want 2 after fingerprint bump", calls)
+	}
+}
+
+func TestChatWithReplayDoesNotCacheErrors(t *testing.T) {
+	dir := t.TempDir()
+	t.Setenv(replayCacheDirEnv, dir)
+
+	calls := 0
+	client := &countingClient{n: &calls, err: errors.New("boom")}
+	for i := 0; i < 2; i++ {
+		if _, err := chatWithReplay(context.Background(), client,
+			[]types.EyrieMessage{{Role: "user", Content: "e"}}, types.ChatOptions{}); err == nil {
+			t.Fatal("expected error to pass through")
+		}
+	}
+	if calls != 2 {
+		t.Fatalf("errors must not be cached; calls = %d", calls)
+	}
+	entries, _ := filepath.Glob(filepath.Join(dir, "resp", "*", "*.json"))
+	if len(entries) != 0 {
+		t.Fatalf("no entries expected after failed calls, got %v", entries)
+	}
+}
+
+type countingClient struct {
+	n    *int
+	resp string
+	err  error
+}
+
+func (c *countingClient) Chat(context.Context, []types.EyrieMessage, types.ChatOptions) (*types.EyrieResponse, error) {
+	*c.n++
+	if c.err != nil {
+		return nil, c.err
+	}
+	return &types.EyrieResponse{Content: c.resp}, nil
+}
+
+func (c *countingClient) StreamChatContinue(context.Context, []types.EyrieMessage, types.ChatOptions, types.ContinuationConfig) (*types.StreamResult, error) {
+	*c.n++
+	if c.err != nil {
+		return nil, c.err
+	}
+	ch := make(chan types.EyrieStreamEvent, 1)
+	ch <- types.EyrieStreamEvent{Type: "done"}
+	return &types.StreamResult{Events: ch}, nil
+}

internal/engine/chat_service.go

@@ -309,7 +309,7 @@ func (c *ChatService) Chat(ctx context.Context, messages []types.EyrieMessage, o
 	if client == nil {
 		return nil, errors.New("chat service: no client configured")
 	}
-	return client.Chat(ctx, messages, opts)
+	return chatWithReplay(ctx, client, messages, opts)
 }
 
 // isContextOverflow reports whether err looks like a "context too long"

internal/engine/stream_exhaustion.go

@@ -0,0 +1,60 @@
+package engine
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/GrayCodeAI/hawk/internal/types"
+)
+
+// synthesisTailMessages is how many recent messages feed the exhaustion prompt.
+const synthesisTailMessages = 8
+
+// SynthesisForExhaustion produces a coherent final completion when the agent
+// loop exhausts its budget (turn/token/time limits), using ONE final
+// tools-disabled LLM call — the "graceful exhaustion" idea from herm. Instead
+// of stopping with a bare "limit reached" line, the model synthesizes a concise
+// summary of what was done, what remains, and next steps. Returns "" when the
+// session has no LLM, no conversation, the call fails, or the context is
+// cancelled — callers fall back to their static stop message.
+func (s *Session) SynthesisForExhaustion(ctx context.Context, reason string) string {
+	if s == nil || s.ChatLLM() == nil {
+		return ""
+	}
+	raw := s.Persistence().RawMessages()
+	if len(raw) == 0 {
+		return ""
+	}
+	if err := ctx.Err(); err != nil {
+		return ""
+	}
+
+	var b strings.Builder
+	b.WriteString("You are completing a coding-agent session that must stop now because its execution budget is exhausted.\n")
+	fmt.Fprintf(&b, "Reason: %s\n\n", reason)
+	b.WriteString("Do not call any tools. Write a concise final message covering: what was accomplished, what is left to do, and suggested next steps.\n\nRecent conversation (most recent first):\n")
+	start := 0
+	if len(raw) > synthesisTailMessages {
+		start = len(raw) - synthesisTailMessages
+	}
+	for i := len(raw) - 1; i >= start; i-- {
+		m := raw[i]
+		fmt.Fprintf(&b, "[%s] %s\n", m.Role, truncateRunes(m.Content, 300))
+	}
+
+	callCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	resp, err := s.ChatLLM().Chat(callCtx, []types.EyrieMessage{
+		{Role: "user", Content: b.String()},
+	}, types.ChatOptions{
+		Provider:  s.ChatLLM().Provider(),
+		Model:     s.ChatLLM().Model(),
+		MaxTokens: 800,
+	})
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(resp.Content)
+}

internal/engine/stream_exhaustion_test.go

@@ -0,0 +1,76 @@
+package engine
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/GrayCodeAI/hawk/internal/types"
+)
+
+func TestSynthesisForExhaustion(t *testing.T) {
+	sess := NewSessionWithClient(NewMockClientForTest(), "test", "test-model", "", nil, false)
+	sess.Persistence().SetRawMessages([]types.EyrieMessage{
+		{Role: "user", Content: "fix the bug"},
+		{Role: "assistant", Content: "I found it"},
+	})
+	out := sess.SynthesisForExhaustion(context.Background(), "turn limit reached")
+	if strings.TrimSpace(out) != "mock test response" {
+		t.Fatalf("got %q, want mock test response", out)
+	}
+}
+
+func TestSynthesisForExhaustionNoSession(t *testing.T) {
+	var nilSess *Session
+	if out := nilSess.SynthesisForExhaustion(context.Background(), "x"); out != "" {
+		t.Fatalf("nil session should return empty, got %q", out)
+	}
+}
+
+func TestSynthesisForExhaustionNoMessages(t *testing.T) {
+	sess := NewSessionWithClient(NewMockClientForTest(), "test", "test-model", "", nil, false)
+	if out := sess.SynthesisForExhaustion(context.Background(), "x"); out != "" {
+		t.Fatalf("empty conversation should return empty, got %q", out)
+	}
+}
+
+func TestSynthesisForExhaustionCancelled(t *testing.T) {
+	sess := NewSessionWithClient(NewMockClientForTest(), "test", "test-model", "", nil, false)
+	sess.Persistence().SetRawMessages([]types.EyrieMessage{{Role: "user", Content: "hi"}})
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if out := sess.SynthesisForExhaustion(ctx, "x"); out != "" {
+		t.Fatalf("cancelled ctx should return empty, got %q", out)
+	}
+}
+
+func TestEmitExhaustionFallsBack(t *testing.T) {
+	// Default (opt-in off): static fallback message even with a live LLM.
+	sess := NewSessionWithClient(NewMockClientForTest(), "test", "test-model", "", nil, false)
+	sess.Persistence().SetRawMessages([]types.EyrieMessage{{Role: "user", Content: "hi"}})
+	ch := make(chan StreamEvent, 2)
+	sess.emitExhaustion(context.Background(), ch, "reason")
+	ev := <-ch
+	if ev.Type != "content" || !strings.Contains(ev.Content, "Limit reached") {
+		t.Fatalf("expected fallback content by default, got %+v", ev)
+	}
+	ev2 := <-ch
+	if ev2.Type != "done" {
+		t.Fatalf("expected done event, got %+v", ev2)
+	}
+}
+
+func TestEmitExhaustionSynthesizesWhenOptedIn(t *testing.T) {
+	t.Setenv(gracefulExhaustionEnv, "1")
+	sess := NewSessionWithClient(NewMockClientForTest(), "test", "test-model", "", nil, false)
+	sess.Persistence().SetRawMessages([]types.EyrieMessage{{Role: "user", Content: "hi"}})
+	ch := make(chan StreamEvent, 2)
+	sess.emitExhaustion(context.Background(), ch, "turn limit reached")
+	ev := <-ch
+	if ev.Type != "content" || !strings.Contains(ev.Content, "mock test response") {
+		t.Fatalf("expected synthesized content when opted in, got %+v", ev)
+	}
+	if ev2 := <-ch; ev2.Type != "done" {
+		t.Fatalf("expected done event, got %+v", ev2)
+	}
+}

internal/engine/stream_guards.go

@@ -4,10 +4,16 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 
 	"github.com/GrayCodeAI/hawk/internal/engine/branching"
 )
 
+// gracefulExhaustionEnv opts the guard path into exhaustion synthesis. Off by
+// default: synthesis is a blocking provider call and the default loop must
+// stop immediately when limits are hit.
+const gracefulExhaustionEnv = "HAWK_GRACEFUL_EXHAUSTION"
+
 // checkGuardConditions runs all pre-turn guard checks.
 // Returns false when the loop should stop (abort conditions met).
 // On first-stage loop detection, injects a break-loop message and continues.
@@ -41,22 +47,49 @@ func (s *Session) checkGuardConditions(ctx context.Context, ch chan<- StreamEven
 
 	if s.LifecycleSvc().Limits() != nil {
 		if exceeded, reason := s.LifecycleSvc().Limits().IsExceeded(); exceeded {
-			ch <- StreamEvent{Type: "content", Content: fmt.Sprintf("\n\nLimit reached: %s", reason)}
-			ch <- StreamEvent{Type: "done"}
+			s.emitExhaustion(ctx, ch, reason)
 			return false
 		}
 	}
 	if allowed, reason := s.tokUsageCanProceed(); !allowed {
-		ch <- StreamEvent{Type: "content", Content: fmt.Sprintf("\n\nLimit reached: %s", reason)}
-		ch <- StreamEvent{Type: "done"}
+		s.emitExhaustion(ctx, ch, reason)
 		return false
 	}
 
 	if s.LifecycleSvc() != nil && s.LifecycleSvc().Limits().MaxTurns() > 0 && turnCount >= s.LifecycleSvc().Limits().MaxTurns() {
-		ch <- StreamEvent{Type: "content", Content: "Turn limit reached — stopping."}
-		ch <- StreamEvent{Type: "done"}
+		s.emitExhaustion(ctx, ch, "turn limit reached")
 		return false
 	}
 
 	return true
 }
+
+// emitExhaustion stops the loop with a graceful completion: one final
+// tools-disabled LLM call synthesizes a summary of the work (herm's graceful
+// exhaustion). Falls back to a static "limit reached" message when synthesis is
+// unavailable or fails.
+//
+// Synthesis performs a blocking provider call, so it is opt-in via
+// HAWK_GRACEFUL_EXHAUSTION=1: the guard path must stay fast and non-blocking
+// by default (a stalled guard delays every terminal event downstream).
+func (s *Session) emitExhaustion(ctx context.Context, ch chan<- StreamEvent, reason string) {
+	if gracefulExhaustionEnabled() {
+		if synth := s.SynthesisForExhaustion(ctx, reason); synth != "" {
+			ch <- StreamEvent{Type: "content", Content: "\n\n" + synth}
+			ch <- StreamEvent{Type: "done"}
+			return
+		}
+	}
+	ch <- StreamEvent{Type: "content", Content: fmt.Sprintf("\n\nLimit reached: %s", reason)}
+	ch <- StreamEvent{Type: "done"}
+}
+
+// gracefulExhaustionEnabled reports whether opt-in exhaustion synthesis is on.
+func gracefulExhaustionEnabled() bool {
+	switch os.Getenv(gracefulExhaustionEnv) {
+	case "1", "true", "TRUE", "True":
+		return true
+	default:
+		return false
+	}
+}