From cb77923bcec55e4f7ad465b65a83ba099dd769b4 Mon Sep 17 00:00:00 2001 From: Evaline Ju <69598118+evaline-ju@users.noreply.github.com> Date: Fri, 20 Nov 2026 14:32:42 -0700 Subject: [PATCH 1/4] :white_check_mark: Wait on events in timer, timeout, backoff tests Signed-off-by: Evaline Ju <69598118+evaline-ju@users.noreply.github.com> --- cmd/agentop/cmd_service.go | 4 ++ cmd/agentop/cmd_service_characterize_test.go | 2 + cmd/agentop/cmd_service_control_test.go | 4 ++ cmd/agentop/cmd_service_platform.go | 15 ++++- cmd/agentop/cmd_service_test.go | 22 ++++++- cmd/agentop/cmd_service_updater_test.go | 3 + cmd/agentop/codex_daemon.go | 6 +- cmd/agentop/codex_daemon_test.go | 22 ++++++- cmd/agentop/local_endpoint_test.go | 26 +++++++- cmd/agentop/server_probe_test.go | 63 ++++++++++++++++--- cmd/agentop/setup_port_preflight_test.go | 21 ++++++- cmd/agentop/setup_prior_test.go | 3 + cmd/agentop/setup_rollback_test.go | 20 +++--- cmd/agentop/setup_step_service_test.go | 8 +-- core/cost/ledger/writer.go | 6 ++ core/cost/ledger/writer_test.go | 14 ++--- .../forwardproxy/truncated_stream_test.go | 2 +- core/listener/httpx/teardown_test.go | 12 ++-- core/pipeline/finisher_test.go | 19 +++--- core/pipeline/snapshot_test.go | 4 +- core/plugins/sessionbudget/e2e_test.go | 12 ++-- core/plugins/sessionbudget/plugin_test.go | 2 +- core/session/archive/archive.go | 15 ++++- core/session/archive/archive_test.go | 6 +- core/session/archive/prior_test.go | 8 +-- core/sessionapi/ingest_test.go | 6 +- core/sessionapi/timeline_test.go | 18 +++--- core/tlsbridge/decision_test.go | 36 ++++++++--- praxis/filters/src/record/emitter.rs | 5 ++ 29 files changed, 281 insertions(+), 103 deletions(-) diff --git a/cmd/agentop/cmd_service.go b/cmd/agentop/cmd_service.go index 91a357e47..cd6638984 100644 --- a/cmd/agentop/cmd_service.go +++ b/cmd/agentop/cmd_service.go @@ -88,6 +88,10 @@ var serviceReadyTimeout = 15 * time.Second // out. var serviceBootoutTimeout = 30 * time.Second +// bootstrapRetryDelay is the pause before a retried launchctl bootstrap, times the +// attempt. A var so a test of a bootstrap that always fails need not wait it out. +var bootstrapRetryDelay = time.Second + // servicePaths is everything the platform-specific bits need, gathered so tests // can point all of it at a temp dir instead of the real ~/Library/LaunchAgents. type servicePaths struct { diff --git a/cmd/agentop/cmd_service_characterize_test.go b/cmd/agentop/cmd_service_characterize_test.go index 22985be72..566be84b2 100644 --- a/cmd/agentop/cmd_service_characterize_test.go +++ b/cmd/agentop/cmd_service_characterize_test.go @@ -440,6 +440,7 @@ const ( // session-history lines must not print. TestCharacterize_ServiceLifecycle's health // endpoint answers from the start, so it pins the other branch. func TestCharacterize_ServiceInstall_NothingServingYet(t *testing.T) { + readyOnFirstAnswer(t) loaded := fakeSupervisor(t) sc := newServiceSceneServing(t, loaded) r := sc.install(t, true, false) @@ -465,6 +466,7 @@ const ( // --restart on an install that is already current must restart rather than report // "Already current". func TestCharacterize_ServiceInstall_RestartWhenCurrent(t *testing.T) { + readyOnFirstAnswer(t) loaded := fakeSupervisor(t) sc := newServiceSceneServing(t, loaded) if r := sc.install(t, true, false); r.code != 0 { diff --git a/cmd/agentop/cmd_service_control_test.go b/cmd/agentop/cmd_service_control_test.go index 9ee329ebd..b1974eb26 100644 --- a/cmd/agentop/cmd_service_control_test.go +++ b/cmd/agentop/cmd_service_control_test.go @@ -66,6 +66,7 @@ func withBuiltinHealthURL(t *testing.T, url string) { // the command: "Restarted." over a proxy that had not been running, and a bare // "Restarted." at all, are what #1282 reported. func TestServiceControl_SaysWhatHappenedToTheProxy(t *testing.T) { + readyOnFirstAnswer(t) loaded := fakeSupervisor(t) sc := newServiceSceneServing(t, loaded) sc.writeUnit(t) @@ -86,6 +87,7 @@ func TestServiceControl_SaysWhatHappenedToTheProxy(t *testing.T) { // the config it last loaded. A stop needs no config, and still works. A missing // config is worded as install words it, not as a file to fix. func TestServiceControl_RefusesToStartOntoABrokenConfig(t *testing.T) { + readyOnFirstAnswer(t) for _, tc := range []struct { name string missing bool @@ -130,6 +132,7 @@ func TestServiceControl_RefusesToStartOntoABrokenConfig(t *testing.T) { // anything was serving, and silence there is not a no: the file may have moved // health_addr. So the stop says what is now true rather than "was not running". func TestServiceControl_StopCannotTellWithABrokenConfig(t *testing.T) { + readyOnFirstAnswer(t) loaded := fakeSupervisor(t) sc := newServiceSceneServing(t, loaded) sc.writeUnit(t) @@ -198,6 +201,7 @@ func statusScene(t *testing.T, missing bool) servicePaths { // built-in health address, and exits 1 either way: the file needs fixing. Silence at // that address is reported as only that, since the file may have moved health_addr. func TestServiceStatus_BrokenConfig(t *testing.T) { + readyOnFirstAnswer(t) status := func(t *testing.T, p servicePaths, health string) string { t.Helper() withBuiltinHealthURL(t, health) diff --git a/cmd/agentop/cmd_service_platform.go b/cmd/agentop/cmd_service_platform.go index 2b6e89951..bbb3a7135 100644 --- a/cmd/agentop/cmd_service_platform.go +++ b/cmd/agentop/cmd_service_platform.go @@ -225,7 +225,7 @@ func loadService(goos string, p servicePaths, progress io.Writer) error { if attempt >= 3 || !strings.Contains(string(out), "Input/output error") { return fmt.Errorf("launchctl bootstrap failed: %v: %s", err, strings.TrimSpace(string(out))) } - time.Sleep(time.Duration(attempt) * time.Second) + time.Sleep(time.Duration(attempt) * bootstrapRetryDelay) if !waitBootedOutf(target, serviceBootoutTimeout, progress) { return fmt.Errorf("launchctl bootstrap failed and the label is still "+ "registered: %v: %s", err, strings.TrimSpace(string(out))) @@ -594,10 +594,17 @@ func stopPID(pid int) error { // in agentop. Tests set it, so that a stub that answers soon is seen soon. var healthPollDivisor time.Duration = 1 +// healthFirstAnswer makes waitHealthy take the first answer, whatever its window. It +// is false in agentop. A test whose stubs answer at once sets it, so that a stub that +// never answers 2xx is not waited out. +var healthFirstAnswer bool + +// waitHealthy reports whether url answers 2xx within the window. It always asks once, +// so a window that is already over takes the first answer. func waitHealthy(url string, within time.Duration) bool { c := &http.Client{Timeout: 2 * time.Second} deadline := time.Now().Add(within) - for time.Now().Before(deadline) { + for { resp, err := c.Get(url) //nolint:noctx // bounded by Timeout if err == nil { _ = resp.Body.Close() @@ -605,9 +612,11 @@ func waitHealthy(url string, within time.Duration) bool { return true } } + if healthFirstAnswer || !time.Now().Before(deadline) { + return false + } time.Sleep(500 * time.Millisecond / healthPollDivisor) } - return false } func lastLines(path string, n int) []string { diff --git a/cmd/agentop/cmd_service_test.go b/cmd/agentop/cmd_service_test.go index a3c7563c3..84d07247b 100644 --- a/cmd/agentop/cmd_service_test.go +++ b/cmd/agentop/cmd_service_test.go @@ -490,6 +490,7 @@ func TestServiceUsage_NamesTheAgentsThatDependOnTheProxy(t *testing.T) { // is for: active, yet silent. Asking the real systemd made the verdict depend on the // machine the test ran on (a CI runner reported "stopped, but still enabled"). func TestServiceStatus_NotAnsweringNamesTheAgentsThatFail(t *testing.T) { + readyOnFirstAnswer(t) saved := unitStateOf unitStateOf = func() unitState { return unitState{known: true, load: "loaded", active: "active", sub: "running", enabled: "enabled", result: "success"} @@ -513,10 +514,27 @@ func TestServiceStatus_NotAnsweringNamesTheAgentsThatFail(t *testing.T) { // TestWaitHealthy_TimesOut: install claims success only after the health endpoint // answers, because a supervisor reports "loaded" for a proxy that is crash-looping. // Within its window it asks again, past a dropped connection and a 503, until the -// answer is 2xx. +// answer is 2xx. A window already over takes the first answer. func TestWaitHealthy_TimesOut(t *testing.T) { if waitHealthy("http://127.0.0.1:1/healthz", time.Nanosecond) { - t.Error("reported healthy with its window already over") + t.Error("reported healthy with nothing listening") + } + var status atomic.Int32 + var asks atomic.Int32 + once := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + asks.Add(1) + w.WriteHeader(int(status.Load())) + })) + t.Cleanup(once.Close) + for _, tc := range []struct { + status int32 + want bool + }{{http.StatusServiceUnavailable, false}, {http.StatusOK, true}} { + status.Store(tc.status) + asks.Store(0) + if got := waitHealthy(once.URL+"/healthz", time.Nanosecond); got != tc.want || asks.Load() != 1 { + t.Errorf("an over window against %d: waitHealthy = %v after %d asks, want %v after 1", tc.status, got, asks.Load(), tc.want) + } } var calls atomic.Int32 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { diff --git a/cmd/agentop/cmd_service_updater_test.go b/cmd/agentop/cmd_service_updater_test.go index 9720fd614..b134306a5 100644 --- a/cmd/agentop/cmd_service_updater_test.go +++ b/cmd/agentop/cmd_service_updater_test.go @@ -149,6 +149,9 @@ func TestServiceInstallWritesUpdaterUnit(t *testing.T) { // must go with it, or nothing removes it. func TestServiceInstallFailedLoadRemovesUpdaterUnit(t *testing.T) { fakeSupervisor(t) + saved := bootstrapRetryDelay + bootstrapRetryDelay = 0 + t.Cleanup(func() { bootstrapRetryDelay = saved }) sc := newServiceScene(t) // Only the proxy's load fails, so install gets past its preflight and writes both files. installStub(t, "launchctl", `#!/bin/sh diff --git a/cmd/agentop/codex_daemon.go b/cmd/agentop/codex_daemon.go index 2dce7b3be..6ac70643f 100644 --- a/cmd/agentop/codex_daemon.go +++ b/cmd/agentop/codex_daemon.go @@ -60,6 +60,10 @@ const codexDaemonRestartCmd = "codex app-server daemon restart" // the forced stop and the new start. This allows the same, plus a margin. const codexRestartTimeout = 5*time.Minute + 90*time.Second +// codexWaitDelay is how long codexRun reads the command's stderr after it exits. A var +// so a test of a child that keeps the pipe open need not wait it out. +var codexWaitDelay = time.Second + // codexDaemon is a running app-server daemon, as its pid file and the process table // describe it. type codexDaemon struct { @@ -367,7 +371,7 @@ func codexRun(bin, codexHome string, want map[string]string, args ...string) err var stderr bytes.Buffer cmd.Stdout = io.Discard cmd.Stderr = &stderr - cmd.WaitDelay = time.Second + cmd.WaitDelay = codexWaitDelay if err := cmd.Run(); err != nil && !errors.Is(err, exec.ErrWaitDelay) { what := filepath.Base(bin) + " " + strings.Join(args, " ") if ctx.Err() != nil { diff --git a/cmd/agentop/codex_daemon_test.go b/cmd/agentop/codex_daemon_test.go index 74d162550..8625b2df3 100644 --- a/cmd/agentop/codex_daemon_test.go +++ b/cmd/agentop/codex_daemon_test.go @@ -13,6 +13,7 @@ import ( "strings" "syscall" "testing" + "time" "github.com/rossoctl/cortex/core/peerproc" ) @@ -659,6 +660,9 @@ func TestCodexRun(t *testing.T) { t.Setenv(codexHomeEnv, "") t.Setenv("HTTPS_PROXY", cortexProxyURL) home := t.TempDir() + saved := codexWaitDelay + t.Cleanup(func() { codexWaitDelay = saved }) + codexWaitDelay = time.Hour // stderr is read to its end err := codexRun("/bin/sh", home, nil, "-c", `echo "home=$CODEX_HOME proxy=$HTTPS_PROXY" >&2; echo second >&2; exit 3`) if err == nil { t.Fatal("a non-zero exit returned no error") @@ -671,12 +675,24 @@ func TestCodexRun(t *testing.T) { } // A restart leaves a daemon running, and a daemon that kept the command's stdout or // stderr open must not turn a successful restart into a failure (#1526 review). The - // background sleep holds both for longer than WaitDelay. - if err := codexRun("/bin/sh", home, nil, "-c", "sleep 5 & exit 0"); err != nil { + // background sleep holds both until the cleanup stops it, so each Wait ends on + // WaitDelay, here already over. + codexWaitDelay = time.Nanosecond + pids := filepath.Join(t.TempDir(), "pids") + t.Cleanup(func() { + b, _ := os.ReadFile(pids) //nolint:gosec // the test's own file + for _, f := range strings.Fields(string(b)) { + if pid, err := strconv.Atoi(f); err == nil { + _ = syscall.Kill(pid, syscall.SIGKILL) + } + } + }) + child := "sleep 3600 & echo $! >> '" + pids + "'; " + if err := codexRun("/bin/sh", home, nil, "-c", child+"exit 0"); err != nil { t.Errorf("a successful command whose child kept its output open returned %v", err) } // The same child behind a failed command is still a failure. - if err := codexRun("/bin/sh", home, nil, "-c", "sleep 5 & exit 4"); err == nil || !strings.Contains(err.Error(), "exit status 4") { + if err := codexRun("/bin/sh", home, nil, "-c", child+"exit 4"); err == nil || !strings.Contains(err.Error(), "exit status 4") { t.Errorf("a failed command whose child kept its output open returned %v, want exit status 4", err) } } diff --git a/cmd/agentop/local_endpoint_test.go b/cmd/agentop/local_endpoint_test.go index 0cde0ad25..7467e088a 100644 --- a/cmd/agentop/local_endpoint_test.go +++ b/cmd/agentop/local_endpoint_test.go @@ -209,11 +209,33 @@ func TestLocalSessionAPIUp_WaitsForAProxyStillStarting(t *testing.T) { _, _ = w.Write([]byte(`{"sessions":[]}`)) })} defer srv.Close() - time.AfterFunc(time.Second, func() { _ = srv.Serve(ln) }) + // The server starts once the client has connected, and serves that connection first. + go func() { + c, err := ln.Accept() + if err != nil { + return + } + _ = srv.Serve(&acceptedFirst{Listener: ln, first: c}) + }() if !localSessionAPIUp("http://" + ln.Addr().String()) { - t.Error("a proxy that started serving 1s after the connection was reported down") + t.Error("a proxy that started serving after the connection was reported down") + } +} + +// acceptedFirst is a listener whose first Accept returns a connection already +// accepted from it. +type acceptedFirst struct { + net.Listener + first net.Conn +} + +func (l *acceptedFirst) Accept() (net.Conn, error) { + if c := l.first; c != nil { + l.first = nil + return c, nil } + return l.Listener.Accept() } // The wait is bounded: something that accepts and never answers is given up on. diff --git a/cmd/agentop/server_probe_test.go b/cmd/agentop/server_probe_test.go index d241a10ff..88daf24f6 100644 --- a/cmd/agentop/server_probe_test.go +++ b/cmd/agentop/server_probe_test.go @@ -6,8 +6,10 @@ import ( "maps" "net/http" "net/http/httptest" + "strconv" "strings" "sync" + "sync/atomic" "testing" "time" @@ -385,27 +387,71 @@ func TestProbe_DroppedPathIsNotTheEnd(t *testing.T) { // hang answers only when the request is cancelled. func hang(r *http.Request) { <-r.Context().Done() } -// probeWithTimeout probes srv as server add does, with per as each request's time. -func probeWithTimeout(srv *httptest.Server, per time.Duration) (map[string]string, string, error) { +// giveUp lets a handler end its own request as the client's per-request timeout +// would: the client gives up a request once its handler calls hang. As a transport +// it is the client's, and each request it sends carries the id hang finds it by. +type giveUp struct { + rt http.RoundTripper + n atomic.Int64 + cancels sync.Map // id → context.CancelFunc +} + +const giveUpHeader = "X-Test-Give-Up" + +func (g *giveUp) RoundTrip(r *http.Request) (*http.Response, error) { + ctx, cancel := context.WithCancel(r.Context()) + id := strconv.FormatInt(g.n.Add(1), 10) + g.cancels.Store(id, cancel) + r = r.Clone(ctx) + r.Header.Set(giveUpHeader, id) + return g.rt.RoundTrip(r) +} + +// hang answers only when the client has given r up, which it does at once. +func (g *giveUp) hang(r *http.Request) { + if cancel, ok := g.cancels.Load(r.Header.Get(giveUpHeader)); ok { + cancel.(context.CancelFunc)() + } + <-r.Context().Done() +} + +// probeGivingUp probes srv as server add does, through g, with each request's time +// out of play: a hung request ends when g gives it up. +func probeGivingUp(srv *httptest.Server, g *giveUp) (map[string]string, string, error) { client := srv.Client() - client.Timeout = per - ctx, cancel := context.WithTimeout(context.Background(), probeTotal(per)) + client.Timeout = time.Hour + g.rt = client.Transport + client.Transport = g + ctx, cancel := context.WithTimeout(context.Background(), probeTotal(client.Timeout)) defer cancel() return probeAPI(ctx, client, routerconfig.Server{URL: srv.URL, Key: "k"}, "m", "", false) } +// The probe's whole time is one request's time for each candidate path, so paths +// that hang still leave each later one its own. +func TestProbeTotal_IsARequestTimeForEachPath(t *testing.T) { + n := 0 + for _, paths := range probePaths { + n += len(paths) + } + if got := probeTotal(time.Minute); got != time.Duration(n)*time.Minute { + t.Errorf("probeTotal(1m) = %s, want %dm", got, n) + } +} + func TestProbe_HungProtocolLeavesTimeForTheNext(t *testing.T) { + g := &giveUp{} srv, _ := fakeProbeServer(t, func(w http.ResponseWriter, p probeReq, r *http.Request) { switch { case strings.HasSuffix(p.path, "messages"): - hang(r) // a Bob-like server: no anthropic path answers + g.hang(r) // a Bob-like server: no anthropic path answers case p.path == "/v1/chat/completions": io.WriteString(w, chatOK) default: http.NotFound(w, nil) } }) - api, _, err := probeWithTimeout(srv, 100*time.Millisecond) + api, _, err := probeGivingUp(srv, g) want := map[string]string{"chat-completions": "/v1/chat/completions"} if err != nil || !maps.Equal(api, want) { t.Errorf("api = %v, err = %v; want %v", api, err, want) @@ -413,17 +459,18 @@ func TestProbe_HungProtocolLeavesTimeForTheNext(t *testing.T) { } func TestProbe_HungPathLeavesTimeForItsProtocol(t *testing.T) { + g := &giveUp{} srv, _ := fakeProbeServer(t, func(w http.ResponseWriter, p probeReq, r *http.Request) { switch p.path { case "/v1/messages": - hang(r) + g.hang(r) case "/anthropic/v1/messages": io.WriteString(w, anthropicOK) default: http.NotFound(w, nil) } }) - api, _, err := probeWithTimeout(srv, 100*time.Millisecond) + api, _, err := probeGivingUp(srv, g) want := map[string]string{"anthropic-messages": "/anthropic/v1/messages"} if err != nil || !maps.Equal(api, want) { t.Errorf("api = %v, err = %v; want %v", api, err, want) diff --git a/cmd/agentop/setup_port_preflight_test.go b/cmd/agentop/setup_port_preflight_test.go index 36e6d8ffa..cc3577db6 100644 --- a/cmd/agentop/setup_port_preflight_test.go +++ b/cmd/agentop/setup_port_preflight_test.go @@ -6,6 +6,8 @@ import ( "path/filepath" "strconv" "strings" + "sync" + "syscall" "testing" ) @@ -99,9 +101,23 @@ func TestPreflightPorts_OurOwnSessionAPIPortStillPasses(t *testing.T) { // (#1545). func TestApplyUnsupervised_CannotStartIsAFailure(t *testing.T) { fakeNoSupervisor(t) + readyOnFirstAnswer(t) // The health endpoint answers 500 forever: the proxy is "alive but never - // healthy", which is the state the false-ready shipped through. - env := serviceEnv(t, func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusInternalServerError) }) + // healthy", which is the state the false-ready shipped through. Its first + // answer waits for the proxy to have logged, on a FIFO the proxy writes next. + logged := filepath.Join(t.TempDir(), "logged") + if err := syscall.Mkfifo(logged, 0o600); err != nil { + t.Fatal(err) + } + var once sync.Once + env := serviceEnv(t, func(w http.ResponseWriter, _ *http.Request) { + once.Do(func() { + if b, err := os.ReadFile(logged); err != nil || len(b) == 0 { //nolint:gosec // the test's own FIFO + t.Errorf("the proxy did not say it had logged: %v", err) + } + }) + w.WriteHeader(http.StatusInternalServerError) + }) freePorts(t) // A cortex whose supervise arm writes the cannot-start line to the log @@ -113,6 +129,7 @@ case "$*" in "--local --write-config") exit 0 ;; "--local --supervise") echo 'level=ERROR msg="cannot start: another process already holds this address" listener=session-api address=127.0.0.1:47601 holder_pid=65100' >&2 + echo logged > '`+logged+`' exec sleep 300 ;; esac `) diff --git a/cmd/agentop/setup_prior_test.go b/cmd/agentop/setup_prior_test.go index 8ab20b7a9..a71e1347d 100644 --- a/cmd/agentop/setup_prior_test.go +++ b/cmd/agentop/setup_prior_test.go @@ -439,6 +439,7 @@ func TestSetupRollbackReportsAServiceItCouldNotUnload(t *testing.T) { // adopts and stops. The supervised one is what was serving, so the rollback // reloads its unit and starts no background copy. func TestSetupRollbackReloadsTheSupervisedCortexOverABackgroundOne(t *testing.T) { + readyOnFirstAnswer(t) sc, running := newUpgradeScene(t) bin := filepath.Join(sc.home, ".local", "bin", "cortex") pidFile := filepath.Join(sc.home, ".cortex", "proxy.pid") @@ -491,6 +492,7 @@ func TestSetupRollbackReloadsTheSupervisedCortexOverABackgroundOne(t *testing.T) // health check. That is the background Cortex, which runServiceInstall adopts and // stops, so the rollback starts it again and leaves the job stopped. func TestSetupRollbackRestartsTheBackgroundCortexBesideAStoppedService(t *testing.T) { + readyOnFirstAnswer(t) running := filepath.Join(t.TempDir(), "running") sc := newSetupScene(t, func(w http.ResponseWriter, _ *http.Request) { _, err := os.Stat(running) @@ -591,6 +593,7 @@ func TestSetupRollbackWaitsOutLaunchdTeardown(t *testing.T) { if runtimeGOOS() != "darwin" { t.Skip("launchd's bootout only: systemd's disable --now returns once the unit has stopped") } + readyOnFirstAnswer(t) t.Run("a fresh service, then the background Cortex it replaced", func(t *testing.T) { sc, old := backgroundCortexScene(t, "cortex") bin := filepath.Join(sc.home, ".local", "bin", "cortex") diff --git a/cmd/agentop/setup_rollback_test.go b/cmd/agentop/setup_rollback_test.go index 5769feb60..48c34357f 100644 --- a/cmd/agentop/setup_rollback_test.go +++ b/cmd/agentop/setup_rollback_test.go @@ -10,7 +10,6 @@ import ( "path/filepath" "strings" "testing" - "time" "github.com/rossoctl/cortex/cmd/agentop/checklist" ) @@ -145,23 +144,23 @@ func brokenRelease(t *testing.T, sc setupScene) (string, []byte) { return next, old } -// shortReadyTimeout cuts serviceReadyTimeout for a test whose start never -// answers, which would otherwise wait the full 15s. -func shortReadyTimeout(t *testing.T) { +// readyOnFirstAnswer is for a test whose stubs answer at once: each health probe and +// supervisor ask takes its first answer, so a stub that never answers is not waited +// out. +func readyOnFirstAnswer(t *testing.T) { t.Helper() - saved := serviceReadyTimeout - serviceReadyTimeout = 2 * time.Second - t.Cleanup(func() { serviceReadyTimeout = saved }) + saved, first := serviceReadyTimeout, healthFirstAnswer + serviceReadyTimeout, healthFirstAnswer = 0, true + t.Cleanup(func() { serviceReadyTimeout, healthFirstAnswer = saved, first }) } // An upgrade whose new cortex never answers puts the old one back, serving. func TestSetupFailedUpgradeRevertsToThePreviousVersion(t *testing.T) { - shortReadyTimeout(t) + readyOnFirstAnswer(t) sc, running := newUpgradeScene(t) before := homeFiles(t, sc.home) next, old := brokenRelease(t, sc) - began := time.Now() code, out := sc.run(t, "--from", next, "--yes") if code != 1 { t.Fatalf("a broken upgrade exited %d:\n%s", code, out) @@ -191,14 +190,13 @@ func TestSetupFailedUpgradeRevertsToThePreviousVersion(t *testing.T) { if b, _ := os.ReadFile(running); !bytes.Equal(b, old) { t.Error("the job left loaded after the revert does not run the previous cortex") } - t.Logf("broken-upgrade round trip took %s (bounded by serviceReadyTimeout)", time.Since(began)) } // An upgrade over a Cortex stopped with `agentop service stop` rolls back to it // stopped. The failed install loaded the new job; left loaded, it would undo the // user's stop and run again at the next login. func TestSetupFailedUpgradeLeavesAStoppedServiceStopped(t *testing.T) { - shortReadyTimeout(t) + readyOnFirstAnswer(t) sc, _ := newUpgradeScene(t) sp, err := resolveServicePaths(filepath.Join(sc.home, ".cortex", "config.yaml"), "", filepath.Join(sc.home, ".local", "bin", "cortex")) diff --git a/cmd/agentop/setup_step_service_test.go b/cmd/agentop/setup_step_service_test.go index 2a3f17f09..1eb08e91e 100644 --- a/cmd/agentop/setup_step_service_test.go +++ b/cmd/agentop/setup_step_service_test.go @@ -133,7 +133,7 @@ func TestServiceStepUpgradeUndoRestoresTheUnitAndThePrior(t *testing.T) { // left: service restart while the unit is there, setup itself once the service // undo could not put it back. func TestServiceStepFailedRestoreNamesWhatIsLeft(t *testing.T) { - shortReadyTimeout(t) + readyOnFirstAnswer(t) for _, unitGone := range []bool{false, true} { t.Run(fmt.Sprintf("unit gone %v", unitGone), func(t *testing.T) { fakeSupervisor(t) @@ -784,7 +784,7 @@ func notRunning(t *testing.T, loaded string) { // A failed start's detail is service install's own words, less what the rollback // makes untrue, then the log since the start: not install's tail of the whole log. func TestServiceStepFailureDetailIsWhatThisStartLeft(t *testing.T) { - shortReadyTimeout(t) + readyOnFirstAnswer(t) t.Run("nothing answers", func(t *testing.T) { fakeSupervisor(t) env := serviceEnv(t, func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusServiceUnavailable) }) @@ -856,7 +856,7 @@ func TestServiceStepRestoresThePriorOnlyOnceTheJobWasReplaced(t *testing.T) { // found it, and its manual says how: a stopped job is stopped again, and a loaded // one that was not answering stays loaded. func TestServiceStepUndoLeavesTheJobAsItFoundIt(t *testing.T) { - shortReadyTimeout(t) + readyOnFirstAnswer(t) for _, tc := range []struct { name string loaded, serving, prior bool @@ -910,7 +910,7 @@ func TestServiceStepUndoLeavesTheJobAsItFoundIt(t *testing.T) { // The background restore refuses while the new job is loaded, whether or not it // answers: one crash-looping still fights a background copy for the ports. func TestServiceStepBackgroundRestoreRefusesBesideALoadedUnhealthyJob(t *testing.T) { - shortReadyTimeout(t) + readyOnFirstAnswer(t) loaded := fakeSupervisor(t) env := serviceEnv(t, func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusServiceUnavailable) }) t.Setenv(sleeperEnv, "1") // a restarted copy of the test binary sleeps too diff --git a/core/cost/ledger/writer.go b/core/cost/ledger/writer.go index e0316cd91..36ef44efa 100644 --- a/core/cost/ledger/writer.go +++ b/core/cost/ledger/writer.go @@ -122,6 +122,9 @@ type Writer struct { // restart, so it pins the behaviour that hides money (see Window). Unexported and set // directly by this package's tests, so it is not API; nil on every production path. betweenWindowReads func() + // syncQueued is a TEST SEAM like betweenWindowReads: sync calls it, when non-nil, once its + // barrier is in ops, so a test can tell a waiting Flush from a returned one. + syncQueued func() // writeMu serialises write, which is the ONE-WRITER INVARIANT run and drainAndWrite document — // held in code now rather than only in prose. @@ -1144,6 +1147,9 @@ func (w *Writer) sync() error { case <-w.quit: return errClosedWhileFlushing } + if w.syncQueued != nil { + w.syncQueued() + } select { case err := <-done: return err diff --git a/core/cost/ledger/writer_test.go b/core/cost/ledger/writer_test.go index 1c0da1af9..65bed7332 100644 --- a/core/cost/ledger/writer_test.go +++ b/core/cost/ledger/writer_test.go @@ -1607,16 +1607,16 @@ func TestFlush_WaitsForALateRowAlreadyQueued(t *testing.T) { w.writeMu.Lock() w.Record("s1", costedEvent(t, "gw", "m", 0.10, 10, 5)) - // The writer goroutine takes the late row and waits on writeMu with it. - for len(w.ops) > 0 { - runtime.Gosched() - } - + queued := make(chan struct{}) + w.syncQueued = func() { close(queued) } flushed := make(chan error, 1) go func() { flushed <- w.Flush() }() // Flush either queues its barrier behind the stalled row or returns. - for len(w.ops) == 0 && len(flushed) == 0 { - runtime.Gosched() + select { + case <-flushed: + w.writeMu.Unlock() + t.Fatal("Flush returned before the queued row was written") + case <-queued: } select { case <-flushed: diff --git a/core/listener/forwardproxy/truncated_stream_test.go b/core/listener/forwardproxy/truncated_stream_test.go index 465f4b6e1..b28d88fd1 100644 --- a/core/listener/forwardproxy/truncated_stream_test.go +++ b/core/listener/forwardproxy/truncated_stream_test.go @@ -223,7 +223,7 @@ func TestIdleReader_CutSurfacesSentinel(t *testing.T) { } // Idle cut: nothing arrives, the timer closes the body. - idle := idleReader(pr, 20*time.Millisecond) + idle := idleReader(pr, time.Nanosecond) if _, err := idle.Read(make([]byte, 4)); !errors.Is(err, errIdleCut) { t.Errorf("read error after the idle cutoff = %v, want errIdleCut", err) } diff --git a/core/listener/httpx/teardown_test.go b/core/listener/httpx/teardown_test.go index 52858c3c1..fed0ff05f 100644 --- a/core/listener/httpx/teardown_test.go +++ b/core/listener/httpx/teardown_test.go @@ -31,18 +31,20 @@ func TestTeardownContext(t *testing.T) { }) t.Run("carries a deadline", func(t *testing.T) { + before := time.Now() ctx, release := TeardownContext(context.Background()) + after := time.Now() defer release() deadline, ok := ctx.Deadline() if !ok { t.Fatal("no deadline: detaching removes the only thing that would ever stop this work, so a plugin that blocks holds a goroutine, its buffers and its pctx forever") } - // A window rather than an equality: the clock starts inside the call. The lower bound is - // what matters — a deadline much shorter than TeardownTimeout would cut finalization - // short — and the upper bound catches a unit mix-up. - if left := time.Until(deadline); left <= TeardownTimeout/2 || left > TeardownTimeout { - t.Errorf("time until deadline = %v, want (%v, %v]", left, TeardownTimeout/2, TeardownTimeout) + // Bounds rather than an equality: the clock starts inside the call. The lower bound is + // what matters — a deadline shorter than TeardownTimeout would cut finalization short — + // and the upper bound catches a unit mix-up. + if lo, hi := before.Add(TeardownTimeout), after.Add(TeardownTimeout); deadline.Before(lo) || deadline.After(hi) { + t.Errorf("deadline = %v, want TeardownTimeout after the call, between %v and %v", deadline, lo, hi) } }) diff --git a/core/pipeline/finisher_test.go b/core/pipeline/finisher_test.go index 0e01c8cef..bd0d8a809 100644 --- a/core/pipeline/finisher_test.go +++ b/core/pipeline/finisher_test.go @@ -250,28 +250,25 @@ func TestRunFinish_FreshContextNotCancelled(t *testing.T) { // unblocked. func TestRunFinish_FreshContextHasDeadline(t *testing.T) { var hadDeadline bool - var deadlineRemaining time.Duration + var deadline time.Time f := newFinisher("f", func(ctx context.Context, _ *Context) { - d, ok := ctx.Deadline() - hadDeadline = ok - if ok { - deadlineRemaining = time.Until(d) - } + deadline, hadDeadline = ctx.Deadline() }) - // Pick an override small enough to see but large enough to reliably - // be positive-remaining when OnFinish reads it. - p, err := New([]Plugin{f}, WithFinishTimeout(250*time.Millisecond)) + p, err := New([]Plugin{f}, WithFinishTimeout(time.Hour)) if err != nil { t.Fatalf("New: %v", err) } pctx := &Context{} p.Run(context.Background(), pctx) + before := time.Now() p.RunFinish(context.Background(), pctx, Outcome{FinalAction: OutcomeAllow}) + after := time.Now() if !hadDeadline { t.Fatal("OnFinish ctx had no deadline; expected framework-set timeout") } - if deadlineRemaining <= 0 || deadlineRemaining > 250*time.Millisecond { - t.Errorf("deadline remaining = %v, want in (0, 250ms]", deadlineRemaining) + if deadline.Before(before.Add(time.Hour)) || deadline.After(after.Add(time.Hour)) { + t.Errorf("deadline = %v, want 1h after RunFinish, between %v and %v", + deadline, before.Add(time.Hour), after.Add(time.Hour)) } } diff --git a/core/pipeline/snapshot_test.go b/core/pipeline/snapshot_test.go index 28a2731bf..1f458e0e6 100644 --- a/core/pipeline/snapshot_test.go +++ b/core/pipeline/snapshot_test.go @@ -280,7 +280,7 @@ func TestDurationSince_ZeroStart(t *testing.T) { func TestDurationSince_ValidStart(t *testing.T) { start := time.Now().Add(-50 * time.Millisecond) got := DurationSince(start) - if got < 40*time.Millisecond || got > 5*time.Second { - t.Errorf("DurationSince(50ms ago) = %v, want roughly ≥40ms", got) + if elapsed := time.Since(start); got < 50*time.Millisecond || got > elapsed { + t.Errorf("DurationSince(50ms ago) = %v, want between 50ms and %v", got, elapsed) } } diff --git a/core/plugins/sessionbudget/e2e_test.go b/core/plugins/sessionbudget/e2e_test.go index a80261c21..21cb6373d 100644 --- a/core/plugins/sessionbudget/e2e_test.go +++ b/core/plugins/sessionbudget/e2e_test.go @@ -31,7 +31,9 @@ func newE2EPlugin(t *testing.T, maxTokens int64, store *memStore) *SessionBudget t.Fatalf("Configure: %v", err) } p.store = store - go p.refreshLoop(30 * time.Millisecond) + // The loop runs for its shutdown path; an hour-long interval keeps ticks + // out of the test, and a test that needs a refresh calls p.refreshCache. + go p.refreshLoop(time.Hour) t.Cleanup(func() { close(p.stopCh); <-p.stopped }) return p } @@ -58,7 +60,7 @@ func newE2EPluginPause(t *testing.T, maxTokens int64, store *memStore, webhookUR } p.store = store p.httpClient = &http.Client{Timeout: 0} - go p.refreshLoop(30 * time.Millisecond) + go p.refreshLoop(time.Hour) t.Cleanup(func() { close(p.stopCh); <-p.stopped }) return p } @@ -177,7 +179,7 @@ func TestE2E_LocalCacheEnforcesDuringOutage(t *testing.T) { t.Fatalf("Configure: %v", err) } p.store = &failingStore{} - go p.refreshLoop(30 * time.Millisecond) + go p.refreshLoop(time.Hour) t.Cleanup(func() { close(p.stopCh); <-p.stopped }) p.mu.Lock() @@ -434,7 +436,7 @@ func TestE2E_PauseMode(t *testing.T) { } p.store = newMemStore() p.httpClient = &http.Client{} - go p.refreshLoop(30 * time.Millisecond) + go p.refreshLoop(time.Hour) t.Cleanup(func() { close(p.stopCh); <-p.stopped }) p.mu.Lock() @@ -481,7 +483,7 @@ func TestE2E_ConcurrentDenyOvershootBound(t *testing.T) { t.Fatalf("Configure: %v", err) } p.store = newMemStore() - go p.refreshLoop(30 * time.Millisecond) + go p.refreshLoop(time.Hour) t.Cleanup(func() { close(p.stopCh); <-p.stopped }) // Seed the cache one call below the limit. With no OnResponseFrame calls diff --git a/core/plugins/sessionbudget/plugin_test.go b/core/plugins/sessionbudget/plugin_test.go index b427842cf..22e3a4785 100644 --- a/core/plugins/sessionbudget/plugin_test.go +++ b/core/plugins/sessionbudget/plugin_test.go @@ -1140,7 +1140,7 @@ func TestShutdown_DoubleCloseSafe(t *testing.T) { } rec := &closeRecordingStore{Store: newMemStore()} p.store = rec - go p.refreshLoop(30 * time.Millisecond) + go p.refreshLoop(time.Hour) if err := p.Shutdown(context.Background()); err != nil { t.Fatalf("first Shutdown: %v", err) diff --git a/core/session/archive/archive.go b/core/session/archive/archive.go index 90daed353..ac857aa04 100644 --- a/core/session/archive/archive.go +++ b/core/session/archive/archive.go @@ -59,6 +59,10 @@ type Option func(*Archive) // WithClock replaces the archive's time source, for tests. func WithClock(now func() time.Time) Option { return func(a *Archive) { a.now = now } } +// WithTicks drives the writer's tick from c in place of its own ticker, for tests: the writer +// ticks once per receive, so on an unbuffered c a send returns once the tick before it has run. +func WithTicks(c <-chan time.Time) Option { return func(a *Archive) { a.ticks = c } } + // WithRetentionDays sets how long a segment is kept after its last write. Zero or negative is // the default, 30; past MaxRetentionDays it is clamped. func WithRetentionDays(days int) Option { return func(a *Archive) { a.retainDays = days } } @@ -118,6 +122,7 @@ type Archive struct { idleClose time.Duration maxOpen int tickEvery time.Duration + ticks <-chan time.Time // when set, the writer's tick source in place of a ticker // Test seams: gate holds the writer before each op, and held, when set, is signaled each time // the writer waits there; segHook sees each new segment writer. @@ -548,13 +553,17 @@ func (a *Archive) inWriter(f func()) { func (a *Archive) run() { defer close(a.done) - t := time.NewTicker(a.tickEvery) - defer t.Stop() + ticks := a.ticks + if ticks == nil { + t := time.NewTicker(a.tickEvery) + defer t.Stop() + ticks = t.C + } for { select { case o := <-a.ops: a.take(o) - case <-t.C: + case <-ticks: a.applyOptOuts(false) a.tick() case <-a.stop: diff --git a/core/session/archive/archive_test.go b/core/session/archive/archive_test.go index 457843896..40ea65457 100644 --- a/core/session/archive/archive_test.go +++ b/core/session/archive/archive_test.go @@ -4,7 +4,6 @@ import ( "errors" "os" "path/filepath" - "runtime" "slices" "sync" "syscall" @@ -166,13 +165,12 @@ func TestArchive_ANewProcessOpensANewSegment(t *testing.T) { // than wait, and it may only fill the queue up to the reserve. func TestRecord_NeverBlocksAndCountsDropsWhenFull(t *testing.T) { a := openTest(t, t.TempDir(), newClock(), WithQueueDepth(reserve+6)) + a.held = make(chan struct{}, 1) a.gate = make(chan struct{}) // The writer takes a rename and is held at the gate with it, so the events // below see the queue empty. a.Rekeyed("filler", "filler2") - for len(a.ops) > 0 { - runtime.Gosched() - } + <-a.held evs := synthSession(45, 34, 512) // 102 events newRecorder(a).record("s1", evs...) // Six fit below the reserve. diff --git a/core/session/archive/prior_test.go b/core/session/archive/prior_test.go index 6d5680a5c..63f8c3034 100644 --- a/core/session/archive/prior_test.go +++ b/core/session/archive/prior_test.go @@ -3,7 +3,6 @@ package archive import ( "os" "path/filepath" - "runtime" "testing" "github.com/rossoctl/cortex/core/pipeline" @@ -163,11 +162,10 @@ func TestPrior_ARenameLeftUnderTheOldIdIsNotCountedTwice(t *testing.T) { st.Rekey("X", "Y") }}, {"the rename is lost to a full queue", func(t *testing.T, a *Archive, st *session.Store) { + a.held = make(chan struct{}, 1) a.gate = make(chan struct{}) - a.Rekeyed("filler", "filler2") // the writer takes it and waits at the gate - for len(a.ops) > 0 { - runtime.Gosched() - } + a.Rekeyed("filler", "filler2") + <-a.held // the writer has taken it and waits at the gate for len(a.ops) < cap(a.ops) { a.Rekeyed("filler", "filler2") } diff --git a/core/sessionapi/ingest_test.go b/core/sessionapi/ingest_test.go index 526541dec..148e07cba 100644 --- a/core/sessionapi/ingest_test.go +++ b/core/sessionapi/ingest_test.go @@ -243,7 +243,9 @@ func TestIngest_StoreAssignsSeqAndAt(t *testing.T) { r := newIngestRig(t) ev := map[string]any{"seq": 99, "sessionId": "forged", "at": "2001-01-01T00:00:00Z", "phase": "request", "requestId": "r1", "inference": map[string]any{"model": "m"}} + before := time.Now() r.post(t, nil, line(t, map[string]any{"event": ev})) + after := time.Now() evs := r.events("s1") if len(evs) != 1 { t.Fatalf("got %d rows", len(evs)) @@ -254,8 +256,8 @@ func TestIngest_StoreAssignsSeqAndAt(t *testing.T) { if evs[0].SessionID != "s1" { t.Errorf("sessionId = %q, want s1", evs[0].SessionID) } - if d := time.Since(evs[0].At); d < 0 || d > time.Second { - t.Errorf("at = %v, want within 1s of now", evs[0].At) + if at := evs[0].At; at.Before(before) || at.After(after) { + t.Errorf("at = %v, want between %v and %v, around the post", at, before, after) } } diff --git a/core/sessionapi/timeline_test.go b/core/sessionapi/timeline_test.go index 748bbad03..0bb028e9c 100644 --- a/core/sessionapi/timeline_test.go +++ b/core/sessionapi/timeline_test.go @@ -305,7 +305,8 @@ func TestTimelineFollowsRetentionPruningAResumedSession(t *testing.T) { record(time.Now().Add(-time.Hour), "recent.example") clock.Store(time.Now().UnixNano()) - a, err := archive.Open(root, archive.WithClock(now)) + ticks := make(chan time.Time) + a, err := archive.Open(root, archive.WithClock(now), archive.WithTicks(ticks)) if err != nil { t.Fatal(err) } @@ -327,16 +328,11 @@ func TestTimelineFollowsRetentionPruningAResumedSession(t *testing.T) { } clock.Add(int64(2 * 24 * time.Hour)) - deadline := time.Now().Add(5 * time.Second) - for { - info, _, _ := a.Page("s1", 0, 0, func(*pipeline.SessionEvent) bool { return false }) - if info.OldestSeq > 1 { - break - } - if time.Now().After(deadline) { - t.Fatal("retention did not prune the old segment") - } - time.Sleep(20 * time.Millisecond) + // The second send returns once the first tick, which prunes, has run. + ticks <- time.Time{} + ticks <- time.Time{} + if info, _, _ := a.Page("s1", 0, 0, func(*pipeline.SessionEvent) bool { return false }); info.OldestSeq <= 1 { + t.Fatalf("retention did not prune the old segment: oldestSeq = %d", info.OldestSeq) } store.Append("s1", pipeline.SessionEvent{At: time.Now(), Direction: pipeline.Outbound, Phase: pipeline.SessionRequest, Host: "after.example"}) if hs := hosts(); slices.Contains(hs, "old.example") { diff --git a/core/tlsbridge/decision_test.go b/core/tlsbridge/decision_test.go index 26cd38225..54a1b30f5 100644 --- a/core/tlsbridge/decision_test.go +++ b/core/tlsbridge/decision_test.go @@ -247,6 +247,18 @@ func window(t *testing.T, s *SkipSet, host string) time.Duration { return time.Until(e.expiry) } +// expiryOf returns when host's skip window ends. +func expiryOf(t *testing.T, s *SkipSet, host string) time.Time { + t.Helper() + s.mu.RLock() + defer s.mu.RUnlock() + e, ok := s.m[host] + if !ok { + t.Fatalf("no entry for %q", host) + } + return e.expiry +} + // TestSkipSet_SucceedClears is the signal the set never had. // // It only ever added entries and waited them out, so a client that demonstrably @@ -345,18 +357,20 @@ func expireWindow(t *testing.T, s *SkipSet, host string) { // outcome the backoff's own comment rules out. func TestSkipSet_PinnedHostEscalatesAcrossWindows(t *testing.T) { s := NewSkipSet() + var last time.Time for i := 1; i <= 6; i++ { if s.Contains("h") { t.Fatalf("rejection %d: host is still skipped, so the proxy would not have forged", i) } + last = time.Now() s.Fail("h") if i < 6 { expireWindow(t, s, "h") } } - if got := window(t, s, "h"); got < s.ttl-time.Second { - t.Errorf("window after 6 rejections, each after the last window expired = %v, "+ - "want the %v ceiling; an expired entry must not restart the backoff", got, s.ttl) + if got := expiryOf(t, s, "h"); got.Before(last.Add(s.ttl)) { + t.Errorf("window after 6 rejections, each after the last window expired, ends at %v, "+ + "want the %v ceiling from the last one; an expired entry must not restart the backoff", got, s.ttl) } } @@ -378,9 +392,10 @@ func TestSkipSet_TransientAfterExpiryGetsTheBaseAndKeepsTheCount(t *testing.T) { } expireWindow(t, s, "h") + before := time.Now() s.Fail("h") - if got := window(t, s, "h"); got < s.ttl-time.Second { - t.Errorf("window after the next rejection = %v, want the %v ceiling; the transient "+ + if got := expiryOf(t, s, "h"); got.Before(before.Add(s.ttl)) { + t.Errorf("window after the next rejection ends at %v, want the %v ceiling; the transient "+ "failure reset the count", got, s.ttl) } } @@ -403,9 +418,10 @@ func TestSkipSet_TransientOnALiveWindow(t *testing.T) { } setWindow(t, s, "h", time.Minute) // an earned window, partly elapsed + set := expiryOf(t, s, "h") s.FailTransient("h") - if got := window(t, s, "h"); got > time.Minute || got < time.Minute-time.Second { - t.Errorf("transient failure on a live window with 1m left = %v, want it left at about 1m", got) + if got := expiryOf(t, s, "h"); !got.Equal(set) { + t.Errorf("transient failure on a live window with 1m left moved its end from %v to %v, want it left", set, got) } } @@ -521,10 +537,10 @@ func TestProgramSkipSet_ABurstCountsOnce(t *testing.T) { s.Fail("q") expireWindow(t, s, "q") s.Fail("q") // counted: a 2x window - earned := window(t, s, "q") + earned := expiryOf(t, s, "q") s.Fail("q") // the rest of that connection's burst - if got := window(t, s, "q"); got < earned-time.Second { - t.Errorf("a burst cut the earned window from %v to %v", earned, got) + if got := expiryOf(t, s, "q"); got.Before(earned) { + t.Errorf("a burst cut the earned window's end from %v to %v", earned, got) } } diff --git a/praxis/filters/src/record/emitter.rs b/praxis/filters/src/record/emitter.rs index 702cf4dab..2b1399237 100644 --- a/praxis/filters/src/record/emitter.rs +++ b/praxis/filters/src/record/emitter.rs @@ -53,7 +53,12 @@ pub const MAX_QUEUE_LINES: usize = 4096; pub const MAX_QUEUE_BYTES: usize = 256 << 20; pub const MAX_LINE: usize = 64 << 20; +#[cfg(not(test))] const BACKOFF_MIN: Duration = Duration::from_millis(100); +/// Tests reconnect within a millisecond. Above zero, so doubling still grows +/// the backoff of an emitter whose ingest stays down. +#[cfg(test)] +const BACKOFF_MIN: Duration = Duration::from_millis(1); const BACKOFF_MAX: Duration = Duration::from_secs(5); /// The emitter counters, and the queue size at the time of the call. From 3619d5913f20fa26386864c021cafbede0907623 Mon Sep 17 00:00:00 2001 From: Evaline Ju <69598118+evaline-ju@users.noreply.github.com> Date: Fri, 20 Nov 2026 15:00:37 -0700 Subject: [PATCH 2/4] :bug: Check health deadline Signed-off-by: Evaline Ju <69598118+evaline-ju@users.noreply.github.com> --- cmd/agentop/cmd_service_platform.go | 3 +++ core/session/archive/archive.go | 10 +++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/cmd/agentop/cmd_service_platform.go b/cmd/agentop/cmd_service_platform.go index bbb3a7135..3ec62b65d 100644 --- a/cmd/agentop/cmd_service_platform.go +++ b/cmd/agentop/cmd_service_platform.go @@ -616,6 +616,9 @@ func waitHealthy(url string, within time.Duration) bool { return false } time.Sleep(500 * time.Millisecond / healthPollDivisor) + if !time.Now().Before(deadline) { + return false + } } } diff --git a/core/session/archive/archive.go b/core/session/archive/archive.go index ac857aa04..fc2adaf09 100644 --- a/core/session/archive/archive.go +++ b/core/session/archive/archive.go @@ -563,9 +563,13 @@ func (a *Archive) run() { select { case o := <-a.ops: a.take(o) - case <-ticks: - a.applyOptOuts(false) - a.tick() + case _, ok := <-ticks: + if ok { + a.applyOptOuts(false) + a.tick() + } else { + ticks = nil + } case <-a.stop: a.drain() // From here nothing more is queued or kept (sendLocked and optOutLocked check From 594c1d8bf8be77bbe99e9b502abcd237218a617b Mon Sep 17 00:00:00 2001 From: Evaline Ju <69598118+evaline-ju@users.noreply.github.com> Date: Fri, 20 Nov 2026 16:50:58 -0700 Subject: [PATCH 3/4] :white_check_mark: Address timer test review Signed-off-by: Evaline Ju <69598118+evaline-ju@users.noreply.github.com> --- cmd/agentop/cmd_observe_test.go | 7 ++--- cmd/agentop/cmd_service_test.go | 16 ++++++++++++ cmd/agentop/local_endpoint_test.go | 5 +++- cmd/agentop/server_probe_test.go | 12 ++++++--- cmd/agentop/setup_port_preflight_test.go | 8 +++++- core/plugins/sessionbudget/e2e_test.go | 33 +++++++++++++++++++++++- core/plugins/sessionbudget/plugin.go | 8 ++++-- praxis/filters/src/record/emitter.rs | 2 +- 8 files changed, 78 insertions(+), 13 deletions(-) diff --git a/cmd/agentop/cmd_observe_test.go b/cmd/agentop/cmd_observe_test.go index 2d9e0f0b4..bd9a5c1a5 100644 --- a/cmd/agentop/cmd_observe_test.go +++ b/cmd/agentop/cmd_observe_test.go @@ -555,9 +555,10 @@ func TestMain(m *testing.M) { main() return } - // Poll a stub's health 10 times as often. The windows stay real: a shorter - // window made tests fail on a loaded machine, where a stub can need more than - // 1s to answer or to exit. + // Poll a stub's health 10 times as often. A test that calls readyOnFirstAnswer + // takes the first answer; the rest keep real windows, because a shorter window + // made tests fail on a loaded machine, where a stub can need more than 1s to + // answer or to exit. healthPollDivisor = 10 os.Exit(m.Run()) } diff --git a/cmd/agentop/cmd_service_test.go b/cmd/agentop/cmd_service_test.go index 84d07247b..e6e314784 100644 --- a/cmd/agentop/cmd_service_test.go +++ b/cmd/agentop/cmd_service_test.go @@ -554,6 +554,22 @@ func TestWaitHealthy_TimesOut(t *testing.T) { } } +// A window that ends during the wait between asks sends no more: the 503 is the +// answer, and the 2xx a later ask would get is past the window. +func TestWaitHealthy_NoAskPastTheWindow(t *testing.T) { + var asks atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if asks.Add(1) == 1 { + w.WriteHeader(http.StatusServiceUnavailable) + } + })) + t.Cleanup(srv.Close) + within := 500 * time.Millisecond / healthPollDivisor / 2 // half the wait between asks + if ok := waitHealthy(srv.URL+"/healthz", within); ok || asks.Load() != 1 { + t.Errorf("waitHealthy = %v after %d asks, want false after 1", ok, asks.Load()) + } +} + // TestLastLines_SurfacesTheReason: "installed but not answering" is useless // without the log tail that says why. func TestLastLines_SurfacesTheReason(t *testing.T) { diff --git a/cmd/agentop/local_endpoint_test.go b/cmd/agentop/local_endpoint_test.go index 7467e088a..2e0d9d45e 100644 --- a/cmd/agentop/local_endpoint_test.go +++ b/cmd/agentop/local_endpoint_test.go @@ -205,16 +205,19 @@ func TestLocalSessionAPIUp_WaitsForAProxyStillStarting(t *testing.T) { if err != nil { t.Fatal(err) } + defer ln.Close() srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(`{"sessions":[]}`)) })} defer srv.Close() - // The server starts once the client has connected, and serves that connection first. + // The server starts past the dial bound after the client has connected, and serves + // that connection first. Outlasting that bound is the behaviour under test. go func() { c, err := ln.Accept() if err != nil { return } + time.Sleep(localProbeTimeout + 100*time.Millisecond) _ = srv.Serve(&acceptedFirst{Listener: ln, first: c}) }() diff --git a/cmd/agentop/server_probe_test.go b/cmd/agentop/server_probe_test.go index 88daf24f6..fa1697d30 100644 --- a/cmd/agentop/server_probe_test.go +++ b/cmd/agentop/server_probe_test.go @@ -391,6 +391,7 @@ func hang(r *http.Request) { <-r.Context().Done() } // would: the client gives up a request once its handler calls hang. As a transport // it is the client's, and each request it sends carries the id hang finds it by. type giveUp struct { + t *testing.T rt http.RoundTripper n atomic.Int64 cancels sync.Map // id → context.CancelFunc @@ -407,12 +408,15 @@ func (g *giveUp) RoundTrip(r *http.Request) (*http.Response, error) { return g.rt.RoundTrip(r) } -// hang answers only when the client has given r up, which it does at once. +// hang answers only when the client has given r up, which it does at once. A +// request g did not send fails the test and is answered at once. func (g *giveUp) hang(r *http.Request) { if cancel, ok := g.cancels.Load(r.Header.Get(giveUpHeader)); ok { cancel.(context.CancelFunc)() + <-r.Context().Done() + } else { + g.t.Errorf("%s carries no give-up id g sent", r.URL.Path) } - <-r.Context().Done() } // probeGivingUp probes srv as server add does, through g, with each request's time @@ -440,7 +444,7 @@ func TestProbeTotal_IsARequestTimeForEachPath(t *testing.T) { } func TestProbe_HungProtocolLeavesTimeForTheNext(t *testing.T) { - g := &giveUp{} + g := &giveUp{t: t} srv, _ := fakeProbeServer(t, func(w http.ResponseWriter, p probeReq, r *http.Request) { switch { case strings.HasSuffix(p.path, "messages"): @@ -459,7 +463,7 @@ func TestProbe_HungProtocolLeavesTimeForTheNext(t *testing.T) { } func TestProbe_HungPathLeavesTimeForItsProtocol(t *testing.T) { - g := &giveUp{} + g := &giveUp{t: t} srv, _ := fakeProbeServer(t, func(w http.ResponseWriter, p probeReq, r *http.Request) { switch p.path { case "/v1/messages": diff --git a/cmd/agentop/setup_port_preflight_test.go b/cmd/agentop/setup_port_preflight_test.go index cc3577db6..bdb36204b 100644 --- a/cmd/agentop/setup_port_preflight_test.go +++ b/cmd/agentop/setup_port_preflight_test.go @@ -118,6 +118,11 @@ func TestApplyUnsupervised_CannotStartIsAFailure(t *testing.T) { }) w.WriteHeader(http.StatusInternalServerError) }) + t.Cleanup(func() { // release a handler still waiting on the FIFO before srv.Close waits for it + if f, err := os.OpenFile(logged, os.O_WRONLY|syscall.O_NONBLOCK, 0); err == nil { + _ = f.Close() + } + }) freePorts(t) // A cortex whose supervise arm writes the cannot-start line to the log @@ -134,7 +139,8 @@ case "$*" in esac `) - _, _, err := serviceStep{}.applyUnsupervised(env) + _, u, err := serviceStep{}.applyUnsupervised(env) + t.Cleanup(func() { _ = u.fn() }) // stop the stub proxy if err == nil { t.Fatal("a proxy that logged cannot start was reported as started") } diff --git a/core/plugins/sessionbudget/e2e_test.go b/core/plugins/sessionbudget/e2e_test.go index 21cb6373d..73762fe48 100644 --- a/core/plugins/sessionbudget/e2e_test.go +++ b/core/plugins/sessionbudget/e2e_test.go @@ -166,7 +166,7 @@ func TestE2E_MultiSession(t *testing.T) { // TestE2E_LocalCacheEnforcesDuringOutage confirms that a populated // cache enforces even when the backing store is unreachable. func TestE2E_LocalCacheEnforcesDuringOutage(t *testing.T) { - // Build without starting refreshLoop so we can swap store safely. + // The store is down for the whole test. p := New() cfg, _ := json.Marshal(config{ RedisURL: "mem://test", @@ -239,6 +239,37 @@ func TestE2E_RefreshRecovery(t *testing.T) { } } +// TestE2E_RefreshLoopRefreshesAtEachTick: each tick of the refresh loop brings +// the cache up to the store's counters. +func TestE2E_RefreshLoopRefreshesAtEachTick(t *testing.T) { + store := newMemStore() + p := New() + cfg, _ := json.Marshal(config{ + RedisURL: "mem://test", + MaxTokens: 200, + OnExceed: "deny", + RedisUnavailable: "fail_open", + }) + if err := p.Configure(cfg); err != nil { + t.Fatalf("Configure: %v", err) + } + p.store = store + store.HashIncr(context.Background(), "session-budget:s", "tokens", 180) + p.mu.Lock() + p.cache["s"] = &counters{tokens: 50} + p.mu.Unlock() + + ticks := make(chan time.Time) + go p.refreshOn(ticks) + ticks <- time.Now() + close(p.stopCh) // the loop finishes the tick's refresh before it stops + <-p.stopped + + if got := p.cache["s"].tokens; got != 180 { + t.Errorf("after a tick: tokens = %d, want 180", got) + } +} + // TestE2E_PodRestart verifies that a fresh plugin with an empty cache // hydrates from Redis on the first request in pause mode — no cold-cache // overshoot for sessions already over-budget on Redis. Only pause mode diff --git a/core/plugins/sessionbudget/plugin.go b/core/plugins/sessionbudget/plugin.go index 0a70d3c5a..95ecf6d80 100644 --- a/core/plugins/sessionbudget/plugin.go +++ b/core/plugins/sessionbudget/plugin.go @@ -685,15 +685,19 @@ func (p *SessionBudget) accumulate(sessionID string, delta tokenDelta) { } func (p *SessionBudget) refreshLoop(interval time.Duration) { - defer close(p.stopped) ticker := time.NewTicker(interval) defer ticker.Stop() + p.refreshOn(ticker.C) +} +// refreshOn refreshes the cache at each tick until Shutdown. +func (p *SessionBudget) refreshOn(ticks <-chan time.Time) { + defer close(p.stopped) for { select { case <-p.stopCh: return - case <-ticker.C: + case <-ticks: p.refreshCache() } } diff --git a/praxis/filters/src/record/emitter.rs b/praxis/filters/src/record/emitter.rs index 2b1399237..75e703a6b 100644 --- a/praxis/filters/src/record/emitter.rs +++ b/praxis/filters/src/record/emitter.rs @@ -55,7 +55,7 @@ pub const MAX_LINE: usize = 64 << 20; #[cfg(not(test))] const BACKOFF_MIN: Duration = Duration::from_millis(100); -/// Tests reconnect within a millisecond. Above zero, so doubling still grows +/// Tests back off for 1ms. Above zero, so doubling still grows /// the backoff of an emitter whose ingest stays down. #[cfg(test)] const BACKOFF_MIN: Duration = Duration::from_millis(1); From 6f40d2bc90b1ded2d2f87510cd6c728acf6d7d79 Mon Sep 17 00:00:00 2001 From: Evaline Ju <69598118+evaline-ju@users.noreply.github.com> Date: Fri, 20 Nov 2026 17:12:01 -0700 Subject: [PATCH 4/4] :white_check_mark: End fifo wait Signed-off-by: Evaline Ju <69598118+evaline-ju@users.noreply.github.com> --- cmd/agentop/cmd_observe_test.go | 3 +++ cmd/agentop/cmd_service_platform.go | 3 ++- cmd/agentop/setup_port_preflight_test.go | 27 +++++++++++++++++------- core/plugins/sessionbudget/e2e_test.go | 3 ++- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/cmd/agentop/cmd_observe_test.go b/cmd/agentop/cmd_observe_test.go index bd9a5c1a5..0e6cdc48e 100644 --- a/cmd/agentop/cmd_observe_test.go +++ b/cmd/agentop/cmd_observe_test.go @@ -559,6 +559,9 @@ func TestMain(m *testing.M) { // takes the first answer; the rest keep real windows, because a shorter window // made tests fail on a loaded machine, where a stub can need more than 1s to // answer or to exit. + // + // Tests set package variables like this one, so tests in this package run + // sequentially: none calls t.Parallel. healthPollDivisor = 10 os.Exit(m.Run()) } diff --git a/cmd/agentop/cmd_service_platform.go b/cmd/agentop/cmd_service_platform.go index 3ec62b65d..6bc08b90c 100644 --- a/cmd/agentop/cmd_service_platform.go +++ b/cmd/agentop/cmd_service_platform.go @@ -600,7 +600,8 @@ var healthPollDivisor time.Duration = 1 var healthFirstAnswer bool // waitHealthy reports whether url answers 2xx within the window. It always asks once, -// so a window that is already over takes the first answer. +// so a window that is already over takes the first answer, which can take the 2s +// request timeout. Every caller in agentop passes a positive window. func waitHealthy(url string, within time.Duration) bool { c := &http.Client{Timeout: 2 * time.Second} deadline := time.Now().Add(within) diff --git a/cmd/agentop/setup_port_preflight_test.go b/cmd/agentop/setup_port_preflight_test.go index bdb36204b..8a5d7109e 100644 --- a/cmd/agentop/setup_port_preflight_test.go +++ b/cmd/agentop/setup_port_preflight_test.go @@ -109,20 +109,27 @@ func TestApplyUnsupervised_CannotStartIsAFailure(t *testing.T) { if err := syscall.Mkfifo(logged, 0o600); err != nil { t.Fatal(err) } + gaveUp := make(chan struct{}) var once sync.Once env := serviceEnv(t, func(w http.ResponseWriter, _ *http.Request) { once.Do(func() { - if b, err := os.ReadFile(logged); err != nil || len(b) == 0 { //nolint:gosec // the test's own FIFO - t.Errorf("the proxy did not say it had logged: %v", err) + read := make(chan []byte, 1) + go func() { + b, _ := os.ReadFile(logged) //nolint:gosec // the test's own FIFO + read <- b + }() + select { + case b := <-read: + if len(b) == 0 { + t.Error("the proxy wrote nothing to say it had logged") + } + case <-gaveUp: + t.Error("the proxy did not say it had logged") } }) w.WriteHeader(http.StatusInternalServerError) }) - t.Cleanup(func() { // release a handler still waiting on the FIFO before srv.Close waits for it - if f, err := os.OpenFile(logged, os.O_WRONLY|syscall.O_NONBLOCK, 0); err == nil { - _ = f.Close() - } - }) + t.Cleanup(func() { close(gaveUp) }) // ends a handler's wait before srv.Close waits for it freePorts(t) // A cortex whose supervise arm writes the cannot-start line to the log @@ -140,7 +147,11 @@ esac `) _, u, err := serviceStep{}.applyUnsupervised(env) - t.Cleanup(func() { _ = u.fn() }) // stop the stub proxy + t.Cleanup(func() { // stop the stub proxy + if u.fn != nil { + _ = u.fn() + } + }) if err == nil { t.Fatal("a proxy that logged cannot start was reported as started") } diff --git a/core/plugins/sessionbudget/e2e_test.go b/core/plugins/sessionbudget/e2e_test.go index 73762fe48..d6a34911c 100644 --- a/core/plugins/sessionbudget/e2e_test.go +++ b/core/plugins/sessionbudget/e2e_test.go @@ -166,7 +166,8 @@ func TestE2E_MultiSession(t *testing.T) { // TestE2E_LocalCacheEnforcesDuringOutage confirms that a populated // cache enforces even when the backing store is unreachable. func TestE2E_LocalCacheEnforcesDuringOutage(t *testing.T) { - // The store is down for the whole test. + // The store is down for the whole test, and the loop's hour-long interval + // keeps ticks out of it. p := New() cfg, _ := json.Marshal(config{ RedisURL: "mem://test",