Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,10 @@ Features adopted from open-source agent projects. All are off by default unless
| Atomic install transactions | `internal/installtxn` | Cross-process staged install/remove with rollback. Wired into skill install (atomic `SKILL.md` publish) |
| Stale-lock reclaim | `internal/lockutil` | Race-correct atomic reclaim of O_EXCL lock files with live-restore (ready for O_EXCL lock sites) |
| Test command discovery | `internal/testrunner` | Auto-detect test/verify commands (Go/npm/bun/pnpm/yarn/pytest/cargo) and parse runner output into structured results. Wired into `hawk verify` |
| Circuit breaker | `internal/circuitbreaker` | Closed/open/half-open retry-storm protection with cooldown — generalized primitive (auto-compact style: 3 failures → cooldown → half-open probe) |
| Smart turn routing | `internal/smartrouting` | Deterministic simple/strong turn classifier with fail-toward-strong safety (cheap model for trivial turns) |
| Conversation arc | `internal/conversationarc` | Durable sidecar memory of goals/decisions/milestones/phase with a byte-stable model-visible summary |
| Relevance pruning | `internal/relevanceprune` | Token-budgeted context pruning that keeps relevant history while preserving recent turns/tool calls/errors |

## Usage

Expand Down
123 changes: 123 additions & 0 deletions internal/circuitbreaker/circuitbreaker.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
// Package circuitbreaker provides a small, self-contained circuit breaker used
// to protect repeated operations from retry storms. It is the Go port of
// OpenClaude's auto-compact circuit breaker: after a threshold of consecutive
// failures, the breaker "opens" and skips the operation for a cooldown window,
// then re-arms in a half-open state so a single success closes it and a single
// failure re-opens it. Hawk uses it to stop runaway auto-compaction (an
// irrecoverable prompt_too_long would otherwise retry thousands of times).
package circuitbreaker

import (
"sync"
"time"
)

// State describes the breaker's lifecycle.
type State int

const (
// Closed allows calls; a success keeps it closed.
Closed State = iota
// Open skips calls until the cooldown elapses.
Open
// HalfOpen allows a probe call after the cooldown; one failure re-opens.
HalfOpen
)

// Breaker is a concurrency-safe circuit breaker. The zero value is not usable;
// use New.
type Breaker struct {
maxConsecutiveFailures int
cooldown time.Duration

mu sync.Mutex
consecutiveFailures int
nextRetryAt time.Time
lastFailureAt time.Time
}

// New returns a breaker that opens after maxConsecutiveFailures (must be >= 1)
// consecutive failures, skipping calls for cooldown.
func New(maxConsecutiveFailures int, cooldown time.Duration) *Breaker {
if maxConsecutiveFailures < 1 {
maxConsecutiveFailures = 1
}
if cooldown < 0 {
cooldown = 0
}
return &Breaker{
maxConsecutiveFailures: maxConsecutiveFailures,
cooldown: cooldown,
}
}

// Decision is the outcome of ShouldAllow.
type Decision struct {
// Allow reports whether the caller may proceed with the operation.
Allow bool
// WasHalfOpen is true when the cooldown had elapsed and a probe was admitted
// (so the caller knows one more failure will re-open the breaker).
WasHalfOpen bool
// EffectiveConsecutiveFailures is the count to use for a subsequent failure
// record (the half-open probe is seeded one below the threshold).
EffectiveConsecutiveFailures int
}

// ShouldAllow consults the breaker at time now. When the breaker is open and
// the cooldown has not elapsed, it returns Allow=false (skip). When the
// cooldown has elapsed it admits a probe with WasHalfOpen=true, seeding the
// failure count one below the threshold so a single failure re-opens.
func (b *Breaker) ShouldAllow(now time.Time) Decision {
b.mu.Lock()
defer b.mu.Unlock()
if b.consecutiveFailures < b.maxConsecutiveFailures {
return Decision{Allow: true, EffectiveConsecutiveFailures: b.consecutiveFailures}
}
if b.nextRetryAt.IsZero() && !b.lastFailureAt.IsZero() {
b.nextRetryAt = b.lastFailureAt.Add(b.cooldown)
}
if !b.nextRetryAt.IsZero() && now.Before(b.nextRetryAt) {
return Decision{Allow: false, EffectiveConsecutiveFailures: b.consecutiveFailures}
}
b.consecutiveFailures = b.maxConsecutiveFailures - 1
return Decision{Allow: true, WasHalfOpen: true, EffectiveConsecutiveFailures: b.consecutiveFailures}
}

// RecordSuccess resets the consecutive-failure count, closing the breaker.
func (b *Breaker) RecordSuccess() {
b.mu.Lock()
defer b.mu.Unlock()
b.consecutiveFailures = 0
b.nextRetryAt = time.Time{}
b.lastFailureAt = time.Time{}
}

// RecordFailure increments the consecutive-failure count and records the time
// the cooldown window should start from.
func (b *Breaker) RecordFailure(now time.Time) {
b.mu.Lock()
defer b.mu.Unlock()
b.consecutiveFailures++
b.lastFailureAt = now
b.nextRetryAt = now.Add(b.cooldown)
}

// State reports the current logical state at time now.
func (b *Breaker) State(now time.Time) State {
b.mu.Lock()
defer b.mu.Unlock()
if b.consecutiveFailures < b.maxConsecutiveFailures {
return Closed
}
if !b.nextRetryAt.IsZero() && now.Before(b.nextRetryAt) {
return Open
}
return HalfOpen
}

// ConsecutiveFailures returns the current consecutive-failure count.
func (b *Breaker) ConsecutiveFailures() int {
b.mu.Lock()
defer b.mu.Unlock()
return b.consecutiveFailures
}
88 changes: 88 additions & 0 deletions internal/circuitbreaker/circuitbreaker_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package circuitbreaker

import (
"testing"
"time"
)

func TestClosedAllowsUntilThreshold(t *testing.T) {
b := New(3, 5*time.Minute)
base := time.Now()
for i := 0; i < 3; i++ {
d := b.ShouldAllow(base)
if !d.Allow || d.WasHalfOpen {
t.Fatalf("call %d: allow=%v halfOpen=%v, want allow", i+1, d.Allow, d.WasHalfOpen)
}
b.RecordFailure(base)
}
if b.State(base) != Open {
t.Fatal("expected Open after 3 failures")
}
}

func TestOpenSkipsUntilCooldownElapses(t *testing.T) {
b := New(3, time.Minute)
base := time.Now()
for i := 0; i < 3; i++ {
b.RecordFailure(base)
}
// Within cooldown: skipped.
if d := b.ShouldAllow(base.Add(30 * time.Second)); d.Allow {
t.Fatal("expected skip while open")
}
// After cooldown: half-open probe admitted.
d := b.ShouldAllow(base.Add(2 * time.Minute))
if !d.Allow || !d.WasHalfOpen {
t.Fatalf("expected half-open probe, got allow=%v halfOpen=%v", d.Allow, d.WasHalfOpen)
}
if d.EffectiveConsecutiveFailures != 2 {
t.Fatalf("effective failures = %d, want 2", d.EffectiveConsecutiveFailures)
}
}

func TestHalfOpenSingleFailureReopens(t *testing.T) {
b := New(3, time.Minute)
base := time.Now()
for i := 0; i < 3; i++ {
b.RecordFailure(base)
}
// Half-open probe admitted after cooldown, then fails once.
b.ShouldAllow(base.Add(2 * time.Minute))
b.RecordFailure(base.Add(2 * time.Minute))
if b.State(base.Add(2*time.Minute+1)) != Open {
t.Fatal("expected Open again after half-open failure")
}
}

func TestHalfOpenSuccessCloses(t *testing.T) {
b := New(3, time.Minute)
base := time.Now()
for i := 0; i < 3; i++ {
b.RecordFailure(base)
}
b.ShouldAllow(base.Add(2 * time.Minute))
b.RecordSuccess()
if b.State(base.Add(3*time.Minute)) != Closed {
t.Fatal("expected Closed after half-open success")
}
if b.ConsecutiveFailures() != 0 {
t.Fatalf("consecutive failures = %d, want 0", b.ConsecutiveFailures())
}
}

func TestMinThresholdFloor(t *testing.T) {
if New(0, time.Minute).maxConsecutiveFailures != 1 {
t.Fatal("threshold should floor to 1")
}
}

func TestNegativeCooldownTreatedAsZero(t *testing.T) {
b := New(1, -time.Second)
base := time.Now()
b.RecordFailure(base)
// Zero cooldown: next call is a half-open probe immediately.
d := b.ShouldAllow(base)
if !d.Allow || !d.WasHalfOpen {
t.Fatalf("expected immediate half-open probe, got %+v", d)
}
}
Loading
Loading