Keep a busy daemon through an update, and bring back what a restart stops

client/restart.go

@@ -0,0 +1,30 @@
+package client
+
+import "github.com/enowx/enowx/proto"
+
+// RestartPlan says what replacing the daemon would stop now. A daemon too old to say is
+// judged by its sessions, every terminal among them as it can't tell what runs there.
+func (c *Client) RestartPlan() (proto.RestartPlan, error) {
+	var plan proto.RestartPlan
+	err := c.Call(proto.MethodDaemonRestartPlan, nil, &plan)
+	if !IsUnknownMethod(err) {
+		return plan, err
+	}
+	list, err := c.Sessions()
+	if err != nil {
+		return proto.RestartPlan{}, err
+	}
+	plan = proto.RestartPlan{Unknown: true}
+	for _, s := range list {
+		item := proto.RestartItem{Session: s.ID, Name: s.Name, What: s.Agent}
+		switch {
+		case s.Exited || s.Queued:
+		case s.Agent == "" || s.Agent == "shell":
+			item.What = "a terminal: this daemon can't say what runs in it"
+			plan.Commands = append(plan.Commands, item)
+		case (s.State == proto.StateWorking && !s.StateStale) || s.State == proto.StateWaiting:
+			plan.Agents = append(plan.Agents, item)
+		}
+	}
+	return plan, nil
+}

core/agent/tabrestart.go

@@ -0,0 +1,71 @@
+package agent
+
+import (
+	"path/filepath"
+	"slices"
+	"strings"
+)
+
+// Flags kept of an agent found running in a terminal when it is brought back: its mode,
+// model and permissions. Anything else may be a prompt, already sent, or a secret.
+var (
+	tabValueFlags = []string{"--model", "-m", "--permission-mode", "--effort", "--settings", "--mcp-config",
+		"--append-system-prompt-file", "--add-dir", "--resume", "-r", "--sandbox", "-s", "--ask-for-approval", "-a"}
+	tabBareFlags = []string{"--dangerously-skip-permissions", "--allow-dangerously-skip-permissions",
+		"--dangerously-bypass-approvals-and-sandbox", "--yolo", "--full-auto"}
+	// tabOneShot marks a run that does its work and ends: nothing to bring back.
+	tabOneShot = []string{"-p", "--print", "exec", "run"}
+)
+
+// TabAgentCommand is the command bringing back the agent running as args in a terminal,
+// on its conversation and in its mode; false for what isn't an agent enowX can continue,
+// which ends with the terminal. It never holds a prompt.
+func TabAgentCommand(args []string) ([]string, bool) {
+	if len(args) == 0 {
+		return nil, false
+	}
+	rest := args[1:]
+	preset, ok := findByProgram(args[0])
+	if !ok && len(args) > 1 && slices.Contains([]string{"node", "bun", "deno", "sh", "bash", "python3"}, strings.TrimSuffix(filepath.Base(args[0]), ".exe")) {
+		preset, ok = findByProgram(args[1])
+		rest = args[2:]
+	}
+	if !ok || len(preset.Continue) == 0 {
+		return nil, false
+	}
+	var kept []string
+	resumes := false
+	for i := 0; i < len(rest); i++ {
+		flag, value, hasValue := strings.Cut(rest[i], "=")
+		switch {
+		case slices.Contains(tabOneShot, flag):
+			return nil, false
+		case slices.Contains(tabBareFlags, flag):
+			kept = append(kept, flag)
+		case slices.Contains(tabValueFlags, flag):
+			resumes = resumes || flag == "--resume" || flag == "-r"
+			if hasValue {
+				kept = append(kept, flag, value)
+			} else if i+1 < len(rest) {
+				kept = append(kept, flag, rest[i+1])
+				i++
+			}
+		}
+	}
+	out := []string{preset.Command}
+	if !resumes {
+		out = append(out, preset.Continue...) // first: some are subcommands
+	}
+	return append(out, kept...), true
+}
+
+// findByProgram is the preset whose command program runs.
+func findByProgram(program string) (Preset, bool) {
+	base := strings.TrimSuffix(strings.TrimSuffix(filepath.Base(program), ".exe"), ".js")
+	for _, p := range Presets {
+		if p.Command == base {
+			return p, true
+		}
+	}
+	return Preset{}, false
+}

core/agent/tabrestart_test.go

@@ -0,0 +1,34 @@
+package agent
+
+import (
+	"slices"
+	"strings"
+	"testing"
+)
+
+func TestTabAgentCommand(t *testing.T) {
+	for _, c := range []struct {
+		name string
+		args []string
+		want []string
+		ok   bool
+	}{
+		{"keeps its mode and model, never its prompt", []string{"/usr/local/bin/claude", "--model", "opus", "--permission-mode=plan", "--dangerously-skip-permissions", "fix the bug"},
+			[]string{"claude", "--continue", "--model", "opus", "--permission-mode", "plan", "--dangerously-skip-permissions"}, true},
+		{"its own conversation", []string{"claude", "--resume", "abc-123"}, []string{"claude", "--resume", "abc-123"}, true},
+		{"through node", []string{"node", "/opt/bin/claude", "--effort", "high"}, []string{"claude", "--continue", "--effort", "high"}, true},
+		{"a secret stays out", []string{"claude", "--api-key", "sk-ant-abcdefgh"}, []string{"claude", "--continue"}, true},
+		{"one shot", []string{"claude", "-p", "write the plan"}, nil, false},
+		{"one shot of Codex", []string{"codex", "exec", "write the plan"}, nil, false},
+		{"not an agent", []string{"npm", "run", "dev"}, nil, false},
+		{"nothing", nil, nil, false},
+	} {
+		got, ok := TabAgentCommand(c.args)
+		if ok != c.ok || !slices.Equal(got, c.want) {
+			t.Errorf("%s: %q, %v; want %q", c.name, got, ok, c.want)
+		}
+		if strings.Contains(strings.Join(got, " "), "sk-ant") || strings.Contains(strings.Join(got, " "), "fix the bug") {
+			t.Errorf("%s: kept %q", c.name, got)
+		}
+	}
+}

daemon/logs.go

@@ -14,6 +14,7 @@ const logsSweepEvery = 6 * time.Hour
 
 func init() {
 	slowMethods[proto.MethodDaemonDiagnostics] = true // it reads files
+	slowMethods[proto.MethodDaemonRestartPlan] = true // it samples processes
 }
 
 // daemonLogWriter is where the daemon's logger writes: its log, and its output too

daemon/persist.go

@@ -27,11 +27,16 @@ const (
 
 // record is a session as saved on disk.
 type record struct {
-	ID       string               `json:"id"`
-	Created  time.Time            `json:"created"`
-	Dir      string               `json:"dir"`
-	Command  []string             `json:"command,omitempty"`
-	Relaunch bool                 `json:"relaunch,omitempty"`
+	ID       string    `json:"id"`
+	Created  time.Time `json:"created"`
+	Dir      string    `json:"dir"`
+	Command  []string  `json:"command,omitempty"`
+	Relaunch bool      `json:"relaunch,omitempty"`
+	// What the daemon's stopping noted, for the next one alone: an update of the record
+	// forgets it, as it no longer holds.
+	MidTurn  bool                 `json:"midTurn,omitempty"`      // its agent was at work
+	TabAgent []string             `json:"tabAgent,omitempty"`     // an agent run by hand in its terminal, to bring back
+	Lost     []string             `json:"lost,omitempty"`         // what ran in its terminal, which can't come back
 	Restart  []string             `json:"restart,omitempty"`      // nil with Relaunch: the login shell
 	Convo    string               `json:"conversation,omitempty"` // Meta.Conversation
 	Exited   bool                 `json:"exited,omitempty"`
@@ -196,6 +201,26 @@ func (p *persistence) loadHistory(id string) []byte {
 
 // stopping marks the daemon as stopping: the sessions it hangs up next are
 // relaunched at the next start, not recorded as exited.
+// restartNote is what a daemon's stopping notes of a session.
+type restartNote struct {
+	midTurn  bool
+	tabAgent []string
+	lost     []string
+}
+
+// noteRestart keeps note in session id's record, for the next daemon.
+func (p *persistence) noteRestart(id string, note restartNote) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	r, known := p.records[id]
+	if !known || p.closing {
+		return
+	}
+	r.MidTurn, r.TabAgent, r.Lost = note.midTurn, note.tabAgent, note.lost
+	p.records[id] = r
+	p.saveLocked()
+}
+
 func (p *persistence) stopping() {
 	p.mu.Lock()
 	p.closing = true
@@ -245,6 +270,10 @@ func (s *Server) restore() {
 				Hibernated: r.Hibernated, HibernatedByHand: r.HibernatedByHand, HibernatedAt: r.HibernatedAt, NoAutoHibernate: r.NoAutoHibernate,
 			},
 		}
+		for _, what := range r.Lost {
+			line := "\r\n\x1b[33m── This was running here, and ended when the daemon restarted: " + what + " ──\x1b[0m"
+			spec.History = append(spec.History, line...)
+		}
 		_, dirErr := os.Stat(r.Dir)
 		if r.Exited || !r.Relaunch || dirErr != nil {
 			s.exitsLogged.Store(r.ID, true) // ended with an earlier daemon
@@ -283,6 +312,10 @@ func (s *Server) restore() {
 				spec.Command = thenShell(preset.Argv(l))
 				spec.Meta.Restart = spec.Command
 			}
+			if len(r.TabAgent) > 0 {
+				spec.Command = thenShell(r.TabAgent) // not a bare shell, which what is sent to it would run
+				s.Diag.Event("restart.tabAgent", "session", r.ID, "agent", r.TabAgent[0])
+			}
 			if r.Display {
 				// A new display: the old one went with the daemon.
 				if disp, err := s.displays.start(r.ID); err == nil {
@@ -302,6 +335,7 @@ func (s *Server) restore() {
 			continue
 		}
 		s.persist.update(sess, r.Command)
+		s.restored(sess, r)
 	}
 }
 

daemon/restart.go

@@ -0,0 +1,136 @@
+package daemon
+
+import (
+	"path/filepath"
+	"slices"
+	"strings"
+
+	"github.com/enowx/enowx/core/agent"
+	"github.com/enowx/enowx/core/diag"
+	"github.com/enowx/enowx/core/procstat"
+	"github.com/enowx/enowx/core/session"
+	"github.com/enowx/enowx/proto"
+)
+
+// What a daemon's restart stops, said before it, and brought back after. It ends what
+// sessions run, as their terminals are its own: an update waits while that would cost
+// anything (hostconn), and when it can't wait, what can come back does, exactly.
+
+// carryOnPrompt tells an agent whose turn a restart cut short to go on.
+const carryOnPrompt = "[enowX] The daemon restarted while you were working, which cut your turn short. Carry on from where you stopped. Check first what already ran before repeating anything."
+
+// restartWhatMost bounds a command as listed.
+const restartWhatMost = 200
+
+// restartPlan says what a restart would stop now, and what the last one ended.
+func (s *Server) restartPlan() proto.RestartPlan {
+	plan := proto.RestartPlan{Lost: s.restartLostNow()}
+	for _, sess := range s.sessions.List() {
+		info := sess.Info()
+		if info.Exited || info.Queued {
+			continue
+		}
+		if _, isAgent := agent.Find(info.Agent); isAgent {
+			if atWork(info) {
+				plan.Agents = append(plan.Agents, proto.RestartItem{Session: info.ID, Name: info.Name, What: info.Agent})
+			}
+			continue
+		}
+		for _, args := range s.sessionCommands(info.ID) {
+			plan.Commands = append(plan.Commands, proto.RestartItem{Session: info.ID, Name: info.Name, What: commandWords(args)})
+		}
+	}
+	s.Diag.Event("restart.plan", "agents", len(plan.Agents), "commands", len(plan.Commands), "lost", len(plan.Lost))
+	return plan
+}
+
+// atWork says an agent is mid-turn, or asking: not one whose state went stale, as an
+// interrupted one's does.
+func atWork(info proto.SessionInfo) bool {
+	return info.State == proto.StateWorking && !info.StateStale || info.State == proto.StateWaiting
+}
+
+// sessionCommands are the programs a terminal's shell is running now.
+func (s *Server) sessionCommands(id string) [][]string {
+	own := s.sessionProgram(id)
+	st := &s.statsState
+	var out [][]string
+	seen := map[int]bool{}
+	for pid := range own {
+		for _, child := range st.sampler.Children(pid) {
+			if own[child] || seen[child] {
+				continue
+			}
+			seen[child] = true
+			if args := procstat.Args(child); len(args) > 0 {
+				out = append(out, args)
+			}
+		}
+	}
+	slices.SortFunc(out, func(a, b []string) int { return strings.Compare(strings.Join(a, " "), strings.Join(b, " ")) })
+	return out
+}
+
+// commandWords is a command to list: its secrets masked, its program by name.
+func commandWords(args []string) string {
+	words := append([]string{filepath.Base(args[0])}, args[1:]...)
+	return clip(diag.Scrub(strings.Join(words, " ")), restartWhatMost)
+}
+
+// noteForRestart writes down, as the daemon stops, what can't be read after: the agents
+// mid-turn, and what runs in terminals, which either comes back or is said to be lost.
+func (s *Server) noteForRestart() {
+	for _, sess := range s.sessions.List() {
+		info, meta := sess.Info(), sess.Meta()
+		if info.Exited || info.Queued || !meta.Relaunch {
+			continue
+		}
+		note := restartNote{}
+		if _, isAgent := agent.Find(info.Agent); isAgent {
+			note.midTurn = atWork(info)
+		} else {
+			for _, args := range s.sessionCommands(info.ID) {
+				if back, ok := agent.TabAgentCommand(args); ok && note.tabAgent == nil {
+					note.tabAgent = back
+				} else {
+					note.lost = append(note.lost, commandWords(args))
+				}
+			}
+		}
+		if note.midTurn || note.tabAgent != nil || len(note.lost) > 0 {
+			s.persist.noteRestart(info.ID, note)
+			s.Diag.Event("restart.noted", "session", info.ID, "midTurn", note.midTurn, "tabAgent", note.tabAgent != nil, "lost", len(note.lost))
+		}
+	}
+}
+
+// restored finishes what a restart noted of session sess: an agent mid-turn is told to
+// carry on, once, as it is told only here and its record has forgotten by then.
+func (s *Server) restored(sess *session.Session, r record) {
+	for _, what := range r.Lost {
+		s.restartLostAdd(proto.RestartItem{Session: r.ID, Name: r.Name, What: what})
+	}
+	if !r.MidTurn {
+		return
+	}
+	s.background.start(func() {
+		if err := s.waitSessionReady(sess.ID()); err != nil {
+			s.Diag.Event("restart.carryOn", "session", sess.ID(), "err", err)
+			return
+		}
+		err := s.input(proto.SessionInputParams{ID: sess.ID(), Text: carryOnPrompt, Paste: true, Enter: true})
+		s.Diag.Event("restart.carryOn", "session", sess.ID(), "err", err)
+	})
+}
+
+func (s *Server) restartLostAdd(item proto.RestartItem) {
+	s.restartLostMu.Lock()
+	defer s.restartLostMu.Unlock()
+	s.restartLost = append(s.restartLost, item)
+}
+
+func (s *Server) restartLostNow() []proto.RestartItem {
+	s.restartLostMu.Lock()
+	defer s.restartLostMu.Unlock()
+	return slices.Clone(s.restartLost)
+}