Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions cmd/agentop/cmd_observe_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -555,9 +555,13 @@ func TestMain(m *testing.M) {
main()
return
}
// Poll a stub's health 10 times as often. The windows stay real: a shorter
// window made tests fail on a loaded machine, where a stub can need more than
// 1s to answer or to exit.
// Poll a stub's health 10 times as often. A test that calls readyOnFirstAnswer
// takes the first answer; the rest keep real windows, because a shorter window
// made tests fail on a loaded machine, where a stub can need more than 1s to
// answer or to exit.
//
// Tests set package variables like this one, so tests in this package run
// sequentially: none calls t.Parallel.
healthPollDivisor = 10
os.Exit(m.Run())
}
Expand Down
4 changes: 4 additions & 0 deletions cmd/agentop/cmd_service.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,10 @@ var serviceReadyTimeout = 15 * time.Second
// out.
var serviceBootoutTimeout = 30 * time.Second

// bootstrapRetryDelay is the pause before a retried launchctl bootstrap, times the
// attempt. A var so a test of a bootstrap that always fails need not wait it out.
var bootstrapRetryDelay = time.Second

// servicePaths is everything the platform-specific bits need, gathered so tests
// can point all of it at a temp dir instead of the real ~/Library/LaunchAgents.
type servicePaths struct {
Expand Down
2 changes: 2 additions & 0 deletions cmd/agentop/cmd_service_characterize_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -440,6 +440,7 @@ const (
// session-history lines must not print. TestCharacterize_ServiceLifecycle's health
// endpoint answers from the start, so it pins the other branch.
func TestCharacterize_ServiceInstall_NothingServingYet(t *testing.T) {
readyOnFirstAnswer(t)
loaded := fakeSupervisor(t)
sc := newServiceSceneServing(t, loaded)
r := sc.install(t, true, false)
Expand All @@ -465,6 +466,7 @@ const (
// --restart on an install that is already current must restart rather than report
// "Already current".
func TestCharacterize_ServiceInstall_RestartWhenCurrent(t *testing.T) {
readyOnFirstAnswer(t)
loaded := fakeSupervisor(t)
sc := newServiceSceneServing(t, loaded)
if r := sc.install(t, true, false); r.code != 0 {
Expand Down
4 changes: 4 additions & 0 deletions cmd/agentop/cmd_service_control_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ func withBuiltinHealthURL(t *testing.T, url string) {
// the command: "Restarted." over a proxy that had not been running, and a bare
// "Restarted." at all, are what #1282 reported.
func TestServiceControl_SaysWhatHappenedToTheProxy(t *testing.T) {
readyOnFirstAnswer(t)
loaded := fakeSupervisor(t)
sc := newServiceSceneServing(t, loaded)
sc.writeUnit(t)
Expand All @@ -86,6 +87,7 @@ func TestServiceControl_SaysWhatHappenedToTheProxy(t *testing.T) {
// the config it last loaded. A stop needs no config, and still works. A missing
// config is worded as install words it, not as a file to fix.
func TestServiceControl_RefusesToStartOntoABrokenConfig(t *testing.T) {
readyOnFirstAnswer(t)
for _, tc := range []struct {
name string
missing bool
Expand Down Expand Up @@ -130,6 +132,7 @@ func TestServiceControl_RefusesToStartOntoABrokenConfig(t *testing.T) {
// anything was serving, and silence there is not a no: the file may have moved
// health_addr. So the stop says what is now true rather than "was not running".
func TestServiceControl_StopCannotTellWithABrokenConfig(t *testing.T) {
readyOnFirstAnswer(t)
loaded := fakeSupervisor(t)
sc := newServiceSceneServing(t, loaded)
sc.writeUnit(t)
Expand Down Expand Up @@ -198,6 +201,7 @@ func statusScene(t *testing.T, missing bool) servicePaths {
// built-in health address, and exits 1 either way: the file needs fixing. Silence at
// that address is reported as only that, since the file may have moved health_addr.
func TestServiceStatus_BrokenConfig(t *testing.T) {
readyOnFirstAnswer(t)
status := func(t *testing.T, p servicePaths, health string) string {
t.Helper()
withBuiltinHealthURL(t, health)
Expand Down
19 changes: 16 additions & 3 deletions cmd/agentop/cmd_service_platform.go
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ func loadService(goos string, p servicePaths, progress io.Writer) error {
if attempt >= 3 || !strings.Contains(string(out), "Input/output error") {
return fmt.Errorf("launchctl bootstrap failed: %v: %s", err, strings.TrimSpace(string(out)))
}
time.Sleep(time.Duration(attempt) * time.Second)
time.Sleep(time.Duration(attempt) * bootstrapRetryDelay)
if !waitBootedOutf(target, serviceBootoutTimeout, progress) {
return fmt.Errorf("launchctl bootstrap failed and the label is still "+
"registered: %v: %s", err, strings.TrimSpace(string(out)))
Expand Down Expand Up @@ -594,20 +594,33 @@ func stopPID(pid int) error {
// in agentop. Tests set it, so that a stub that answers soon is seen soon.
var healthPollDivisor time.Duration = 1

// healthFirstAnswer makes waitHealthy take the first answer, whatever its window. It
// is false in agentop. A test whose stubs answer at once sets it, so that a stub that
// never answers 2xx is not waited out.
var healthFirstAnswer bool
Comment thread
evaline-ju marked this conversation as resolved.
Resolved

// waitHealthy reports whether url answers 2xx within the window. It always asks once,
Comment thread
evaline-ju marked this conversation as resolved.
Resolved
// so a window that is already over takes the first answer, which can take the 2s
// request timeout. Every caller in agentop passes a positive window.
func waitHealthy(url string, within time.Duration) bool {
c := &http.Client{Timeout: 2 * time.Second}
deadline := time.Now().Add(within)
for time.Now().Before(deadline) {
for {
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
resp, err := c.Get(url) //nolint:noctx // bounded by Timeout
if err == nil {
_ = resp.Body.Close()
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
return true
}
}
if healthFirstAnswer || !time.Now().Before(deadline) {
return false
}
time.Sleep(500 * time.Millisecond / healthPollDivisor)
if !time.Now().Before(deadline) {
return false
}
}
return false
}

func lastLines(path string, n int) []string {
Expand Down
38 changes: 36 additions & 2 deletions cmd/agentop/cmd_service_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -490,6 +490,7 @@ func TestServiceUsage_NamesTheAgentsThatDependOnTheProxy(t *testing.T) {
// is for: active, yet silent. Asking the real systemd made the verdict depend on the
// machine the test ran on (a CI runner reported "stopped, but still enabled").
func TestServiceStatus_NotAnsweringNamesTheAgentsThatFail(t *testing.T) {
readyOnFirstAnswer(t)
saved := unitStateOf
unitStateOf = func() unitState {
return unitState{known: true, load: "loaded", active: "active", sub: "running", enabled: "enabled", result: "success"}
Expand All @@ -513,10 +514,27 @@ func TestServiceStatus_NotAnsweringNamesTheAgentsThatFail(t *testing.T) {
// TestWaitHealthy_TimesOut: install claims success only after the health endpoint
// answers, because a supervisor reports "loaded" for a proxy that is crash-looping.
// Within its window it asks again, past a dropped connection and a 503, until the
// answer is 2xx.
// answer is 2xx. A window already over takes the first answer.
func TestWaitHealthy_TimesOut(t *testing.T) {
if waitHealthy("http://127.0.0.1:1/healthz", time.Nanosecond) {
t.Error("reported healthy with its window already over")
t.Error("reported healthy with nothing listening")
}
var status atomic.Int32
var asks atomic.Int32
once := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
asks.Add(1)
w.WriteHeader(int(status.Load()))
}))
t.Cleanup(once.Close)
for _, tc := range []struct {
status int32
want bool
}{{http.StatusServiceUnavailable, false}, {http.StatusOK, true}} {
status.Store(tc.status)
asks.Store(0)
if got := waitHealthy(once.URL+"/healthz", time.Nanosecond); got != tc.want || asks.Load() != 1 {
t.Errorf("an over window against %d: waitHealthy = %v after %d asks, want %v after 1", tc.status, got, asks.Load(), tc.want)
}
}
var calls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
Expand All @@ -536,6 +554,22 @@ func TestWaitHealthy_TimesOut(t *testing.T) {
}
}

// A window that ends during the wait between asks sends no more: the 503 is the
// answer, and the 2xx a later ask would get is past the window.
func TestWaitHealthy_NoAskPastTheWindow(t *testing.T) {
var asks atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
if asks.Add(1) == 1 {
w.WriteHeader(http.StatusServiceUnavailable)
}
}))
t.Cleanup(srv.Close)
within := 500 * time.Millisecond / healthPollDivisor / 2 // half the wait between asks
if ok := waitHealthy(srv.URL+"/healthz", within); ok || asks.Load() != 1 {
t.Errorf("waitHealthy = %v after %d asks, want false after 1", ok, asks.Load())
}
}

// TestLastLines_SurfacesTheReason: "installed but not answering" is useless
// without the log tail that says why.
func TestLastLines_SurfacesTheReason(t *testing.T) {
Expand Down
3 changes: 3 additions & 0 deletions cmd/agentop/cmd_service_updater_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,9 @@ func TestServiceInstallWritesUpdaterUnit(t *testing.T) {
// must go with it, or nothing removes it.
func TestServiceInstallFailedLoadRemovesUpdaterUnit(t *testing.T) {
fakeSupervisor(t)
saved := bootstrapRetryDelay
bootstrapRetryDelay = 0
t.Cleanup(func() { bootstrapRetryDelay = saved })
sc := newServiceScene(t)
// Only the proxy's load fails, so install gets past its preflight and writes both files.
installStub(t, "launchctl", `#!/bin/sh
Expand Down
6 changes: 5 additions & 1 deletion cmd/agentop/codex_daemon.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,10 @@ const codexDaemonRestartCmd = "codex app-server daemon restart"
// the forced stop and the new start. This allows the same, plus a margin.
const codexRestartTimeout = 5*time.Minute + 90*time.Second

// codexWaitDelay is how long codexRun reads the command's stderr after it exits. A var
// so a test of a child that keeps the pipe open need not wait it out.
var codexWaitDelay = time.Second

// codexDaemon is a running app-server daemon, as its pid file and the process table
// describe it.
type codexDaemon struct {
Expand Down Expand Up @@ -367,7 +371,7 @@ func codexRun(bin, codexHome string, want map[string]string, args ...string) err
var stderr bytes.Buffer
cmd.Stdout = io.Discard
cmd.Stderr = &stderr
cmd.WaitDelay = time.Second
cmd.WaitDelay = codexWaitDelay
if err := cmd.Run(); err != nil && !errors.Is(err, exec.ErrWaitDelay) {
what := filepath.Base(bin) + " " + strings.Join(args, " ")
if ctx.Err() != nil {
Expand Down
22 changes: 19 additions & 3 deletions cmd/agentop/codex_daemon_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"strings"
"syscall"
"testing"
"time"

"github.com/rossoctl/cortex/core/peerproc"
)
Expand Down Expand Up @@ -659,6 +660,9 @@ func TestCodexRun(t *testing.T) {
t.Setenv(codexHomeEnv, "")
t.Setenv("HTTPS_PROXY", cortexProxyURL)
home := t.TempDir()
saved := codexWaitDelay
t.Cleanup(func() { codexWaitDelay = saved })
codexWaitDelay = time.Hour // stderr is read to its end
err := codexRun("/bin/sh", home, nil, "-c", `echo "home=$CODEX_HOME proxy=$HTTPS_PROXY" >&2; echo second >&2; exit 3`)
if err == nil {
t.Fatal("a non-zero exit returned no error")
Expand All @@ -671,12 +675,24 @@ func TestCodexRun(t *testing.T) {
}
// A restart leaves a daemon running, and a daemon that kept the command's stdout or
// stderr open must not turn a successful restart into a failure (#1526 review). The
// background sleep holds both for longer than WaitDelay.
if err := codexRun("/bin/sh", home, nil, "-c", "sleep 5 & exit 0"); err != nil {
// background sleep holds both until the cleanup stops it, so each Wait ends on
// WaitDelay, here already over.
codexWaitDelay = time.Nanosecond
pids := filepath.Join(t.TempDir(), "pids")
t.Cleanup(func() {
b, _ := os.ReadFile(pids) //nolint:gosec // the test's own file
for _, f := range strings.Fields(string(b)) {
if pid, err := strconv.Atoi(f); err == nil {
_ = syscall.Kill(pid, syscall.SIGKILL)
}
}
})
child := "sleep 3600 & echo $! >> '" + pids + "'; "
if err := codexRun("/bin/sh", home, nil, "-c", child+"exit 0"); err != nil {
t.Errorf("a successful command whose child kept its output open returned %v", err)
}
// The same child behind a failed command is still a failure.
if err := codexRun("/bin/sh", home, nil, "-c", "sleep 5 & exit 4"); err == nil || !strings.Contains(err.Error(), "exit status 4") {
if err := codexRun("/bin/sh", home, nil, "-c", child+"exit 4"); err == nil || !strings.Contains(err.Error(), "exit status 4") {
t.Errorf("a failed command whose child kept its output open returned %v, want exit status 4", err)
}
}
Expand Down
29 changes: 27 additions & 2 deletions cmd/agentop/local_endpoint_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,15 +205,40 @@ func TestLocalSessionAPIUp_WaitsForAProxyStillStarting(t *testing.T) {
if err != nil {
t.Fatal(err)
}
defer ln.Close()
srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`{"sessions":[]}`))
})}
defer srv.Close()
time.AfterFunc(time.Second, func() { _ = srv.Serve(ln) })
// The server starts past the dial bound after the client has connected, and serves
// that connection first. Outlasting that bound is the behaviour under test.
go func() {
c, err := ln.Accept()
if err != nil {
return
}
time.Sleep(localProbeTimeout + 100*time.Millisecond)
_ = srv.Serve(&acceptedFirst{Listener: ln, first: c})
}()

if !localSessionAPIUp("http://" + ln.Addr().String()) {
t.Error("a proxy that started serving 1s after the connection was reported down")
t.Error("a proxy that started serving after the connection was reported down")
}
}

// acceptedFirst is a listener whose first Accept returns a connection already
// accepted from it.
type acceptedFirst struct {
net.Listener
first net.Conn
}

func (l *acceptedFirst) Accept() (net.Conn, error) {
if c := l.first; c != nil {
l.first = nil
return c, nil
}
return l.Listener.Accept()
}

// The wait is bounded: something that accepts and never answers is given up on.
Expand Down
Loading
Loading