Add v1 dashboard with displays each subsystem deployed processes and the ability to start/stop them

.github/workflows/pylint.yml

@@ -23,7 +23,7 @@ jobs:
       - name: Install
         run: |
           python -m pip install -U pip
-          python -m pip install pylint .
+          python -m pip install pylint ".[dashboard]"
 
       - name: Run pylint
         run: pylint src/instrumentctl/

README.md

@@ -68,6 +68,8 @@ common keys with their defaults, ready to copy.
 | `git.email_domain` | the hostname | operators commit as `<user>@<this>` |
 | `legacy.unit` | none | a template unit an older version installed, reported by `doctor` |
 | `legacy.polkit_rule` | none | likewise for its polkit rule |
+| `dashboard.bind` | `127.0.0.1` | where the dashboard listens; `0.0.0.0` opens it to the network |
+| `dashboard.port` | `8750` | likewise |
 
 Which credentials a host needs is deliberately not configured. It is derived
 from the deployed configs and from the daemon code that reads them, so it
@@ -116,6 +118,29 @@ reviewed pull request. Fine-grained tokens expire, at most a year out, and
 `doctor` cannot tell an expired one from a good one, so put the date in a
 calendar.
 
+## Dashboard
+
+`<name> dashboard` serves a page showing every daemon deployed on the host:
+state, restarts, PID, memory, whether the config was edited there, and whether
+the checkout is on a hotfix. It refreshes every two seconds, and has start,
+stop and restart for each instance and each subsystem, which act exactly as
+the CLI's do (a subsystem stops in reverse order).
+
+It needs Flask and waitress, the `dashboard` extra: have the instrument depend
+on `instrumentctl[dashboard]`. `install` renders `<name>-dashboard.service`,
+which runs it as the daemon account; polkit lets that account, as a member of
+the ops group, start and stop the instrument's units and nothing else.
+Whether a host serves it is left to an admin:
+
+```bash
+sudo systemctl enable --now <name>-dashboard
+```
+
+There is no login. Anyone who can reach the page can stop the daemons, so it
+listens on loopback unless `[dashboard] bind` says otherwise, and that should
+only be on a private network. Each action is logged with the address it came
+from: `journalctl -u <name>-dashboard`.
+
 ## Tests
 
 ```bash
@@ -126,6 +151,15 @@ They build a throwaway host tree and exercise real directory reads, stubbing
 only systemd. The rendering tests assert that no `@placeholder@` survives,
 which is what catches a template gaining a variable nobody wired up.
 
+To see the dashboard without systemd, run it against a pretend host whose
+daemons start, stop, crash and restart in memory:
+
+```bash
+uv run --extra test python scripts/dashboard_demo.py    # then open http://127.0.0.1:8750/
+```
+
+`--hotfix` puts the pretend checkout on a hotfix branch, to see those warnings.
+
 ## Host setup
 
 Every instrument installs the same way. The instrument's own docs add only its

instrument.toml.example

@@ -30,3 +30,9 @@ name = "demo"
 # base = "main"                 # the branch hosts run and hotfixes are reviewed into
 # branch_prefix = "hotfix/"
 # email_domain = ""             # operators commit as <user>@<this>; default: the hostname
+
+# `<name> dashboard`, served by <name>-dashboard.service. It has no login and
+# can stop every daemon, so it listens on loopback unless opened up here
+[dashboard]
+# bind = "127.0.0.1"            # "0.0.0.0" for every interface, on a private network
+# port = 8750

pyproject.toml

@@ -13,7 +13,8 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-test = ["pytest"]
+dashboard = ["flask>=3.0", "waitress>=3.0"]
+test = ["pytest", "flask>=3.0", "waitress>=3.0"]
 
 [tool.setuptools.packages.find]
 where = ["src"]

scripts/dashboard_demo.py

@@ -0,0 +1,211 @@
+"""Run the dashboard against a pretend host, to see it without systemd.
+
+    uv run --extra test python scripts/dashboard_demo.py [--port 8750] [--hotfix]
+
+Builds a throwaway host tree (repo checkout, /etc, instance files) in a
+temporary directory and replaces systemctl with an in-memory one, so start,
+stop and restart work and take a moment, as real daemons do. One instance
+crashes every 30 seconds and is restarted, to show what a flapping daemon
+looks like; one starts out failed; one is refused, as polkit would refuse an
+account outside the ops group.
+"""
+from __future__ import annotations
+
+import argparse
+import itertools
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, List, Optional, Sequence
+
+from instrumentctl import dashboard, units
+from instrumentctl.instances import Paths
+from instrumentctl.product import Product
+
+INSTRUMENT_TOML = """\
+[instrument]
+name = "demo"
+display_name = "Demo Spectrograph"
+"""
+
+# name -> state at start; the last is in the repo for this host, not deployed
+INSTANCES = {
+    "power_pdu1": "active",
+    "power_pdu2": "active",
+    "fei_adc": "active",
+    "fei_atc": "active",
+    "fei_fw": "inactive",
+    "cam_ccd": "failed",
+    "guider_tt": None,
+}
+FLAKY = "fei_atc"           # crashes every CRASH_EVERY seconds, then restarts
+REFUSED = "power_pdu2"      # every action on it is denied
+CRASH_EVERY = 30.0
+TRANSITION = 1.5            # seconds a start or stop takes
+RESTART_DELAY = 5.0         # RestartSec= in the real unit
+pids = itertools.count(41000)
+
+
+# One attribute per property the real systemctl show reports
+@dataclass
+class Unit:  # pylint: disable=too-many-instance-attributes
+    """One pretend daemon."""
+
+    active: str
+    since: float
+    sub: str = "dead"
+    restarts: int = 0
+    result: str = "success"
+    exit_status: int = 0
+    pid: int = 0
+    # (state, when): where a start or stop in progress will land
+    pending: Optional[tuple] = None
+
+    def become(self, active: str, now: float) -> None:
+        """Change state, as systemd records it."""
+        self.active, self.since, self.pending = active, now, None
+        self.sub = {"active": "running", "failed": "failed"}.get(active, "dead")
+        self.pid = next(pids) if active == "active" else 0
+
+
+class FakeSystemd:
+    """Stands in for ``units.run``: answers ``show`` and acts on start/stop/restart."""
+
+    def __init__(self, product: Product, initial: Dict[str, str]) -> None:
+        self.product = product
+        self.lock = threading.Lock()
+        now = time.time()
+        self.units = {name: Unit(active="inactive", since=now)
+                      for name in initial}
+        for name, state in initial.items():
+            self.units[name].become(state, now)
+        failed = self.units.get("cam_ccd")
+        if failed:
+            failed.result, failed.exit_status = "exit-code", 1
+
+    def __call__(self, cmd: Sequence[str], capture: bool = False,
+                 prog: str = "") -> subprocess.CompletedProcess:
+        cmd = list(cmd)
+        with self.lock:
+            now = time.time()
+            for name, unit in self.units.items():
+                self._settle(name, unit, now)
+            if cmd[:2] == ["systemctl", "show"]:
+                names = [self.product.instance_of(u) for u in cmd[4:]]
+                text = "\n\n".join(self._show(n) for n in names if n in self.units)
+                return subprocess.CompletedProcess(cmd, 0, text + "\n", "")
+            if cmd[:2] == ["systemctl", "--no-ask-password"]:
+                return self._act(cmd, cmd[2], self.product.instance_of(cmd[3]), now)
+        return subprocess.CompletedProcess(cmd, 1, "", f"demo: cannot fake {cmd}\n")
+
+    @staticmethod
+    def _settle(name: str, unit: Unit, now: float) -> None:
+        if unit.pending and now >= unit.pending[1]:
+            unit.become(unit.pending[0], now)
+        if name == FLAKY and unit.active == "active" and now - unit.since >= CRASH_EVERY:
+            # Restart=on-failure: systemd waits, then tries again
+            unit.become("activating", now)
+            unit.sub, unit.result, unit.exit_status = "auto-restart", "exit-code", 1
+            unit.restarts += 1
+            unit.pending = ("active", now + RESTART_DELAY)
+
+    def _act(self, cmd: List[str], verb: str, name: Optional[str],
+             now: float) -> subprocess.CompletedProcess:
+        if name not in self.units:
+            return subprocess.CompletedProcess(cmd, 5, "", f"Unit {cmd[3]} not found.\n")
+        if name == REFUSED:
+            return subprocess.CompletedProcess(
+                cmd, 1, "", f"Failed to {verb} {cmd[3]}: Access denied\n")
+        unit = self.units[name]
+        if verb == "stop":
+            unit.become("deactivating", now)
+            unit.pending = ("inactive", now + TRANSITION)
+        else:
+            unit.become("activating", now)
+            unit.sub, unit.result, unit.exit_status = "start", "success", 0
+            unit.restarts = 0
+            unit.pending = ("active", now + TRANSITION)
+        return subprocess.CompletedProcess(cmd, 0, "", "")
+
+    def _show(self, name: str) -> str:
+        unit = self.units[name]
+        stamp = time.strftime("%a %Y-%m-%d %H:%M:%S %Z", time.localtime(unit.since))
+        memory = str(30_000_000 + 1_000_000 * len(name)) if unit.pid else "[not set]"
+        return "\n".join([
+            f"Id={self.product.unit(name)}", f"ActiveState={unit.active}",
+            f"SubState={unit.sub}", "UnitFileState=enabled",
+            f"StateChangeTimestamp={stamp}", f"NRestarts={unit.restarts}",
+            f"Result={unit.result}", f"ExecMainStatus={unit.exit_status}",
+            f"MainPID={unit.pid}", f"MemoryCurrent={memory}"])
+
+
+def _git(repo: Path, *args: str) -> None:
+    subprocess.run(["git", "-C", str(repo), "-c", "user.name=demo",
+                    "-c", "user.email=demo@example.invalid", *args],
+                   check=True, capture_output=True)
+
+
+def build_host(root: Path, hotfix: bool) -> Paths:
+    """A checkout, an /etc with most instances deployed, and the instrument.toml."""
+    repo, etc = root / "repo", root / "etc"
+    (repo / "systemd" / "instances").mkdir(parents=True)
+    (etc / "instances").mkdir(parents=True)
+    (etc / "host").write_text("demohost\n", encoding="utf-8")
+    for name, state in INSTANCES.items():
+        subsystem, device = name.split("_", 1)
+        env = repo / "systemd" / "instances" / f"{name}.env"
+        env.write_text(f"DEMO_DAEMON={subsystem}/{device}_daemon.py\n"
+                       "DEMO_HOST=demohost\n", encoding="utf-8")
+        config = repo / "config" / subsystem / f"{name}.yaml"
+        config.parent.mkdir(parents=True, exist_ok=True)
+        config.write_text("hardware:\n  port: 9000\n", encoding="utf-8")
+        if state is not None:
+            (etc / "instances" / env.name).write_bytes(env.read_bytes())
+            (etc / config.name).write_bytes(config.read_bytes())
+    # Hardware values tuned on the host, as the dashboard should point out
+    (etc / "cam_ccd.yaml").write_text("hardware:\n  port: 9001\n", encoding="utf-8")
+
+    _git(repo, "init", "-q", "-b", "main")
+    _git(repo, "add", "-A")
+    _git(repo, "commit", "-q", "-m", "Initial")
+    if hotfix:
+        _git(repo, "update-ref", "refs/remotes/origin/main", "HEAD")
+        _git(repo, "checkout", "-q", "-b", "hotfix/2026-10-27-demohost-adc-hangs")
+        (repo / "config" / "fei" / "fei_adc.yaml").write_text(
+            "hardware:\n  port: 9002\n", encoding="utf-8")
+        _git(repo, "commit", "-q", "-am", "ADC hangs after power cycle")
+        (repo / "notes.txt").write_text("unsaved\n", encoding="utf-8")
+
+    (root / "instrument.toml").write_text(INSTRUMENT_TOML, encoding="utf-8")
+    product = Product.load(root / "instrument.toml")
+    return Paths.from_env(product, environ={
+        "DEMO_REPO_DIR": str(repo), "DEMO_ETC_DIR": str(etc),
+        "DEMO_VENV_DIR": str(root / "venv"), "DEMO_LOGS_DIR": str(root / "logs"),
+    })
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    """Build the pretend host and serve the dashboard until interrupted."""
+    parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
+    parser.add_argument("--bind", default="127.0.0.1")
+    parser.add_argument("--port", type=int, default=8750)
+    parser.add_argument("--hotfix", action="store_true",
+                        help="put the checkout on a hotfix branch with an unsaved edit")
+    args = parser.parse_args(argv)
+    with tempfile.TemporaryDirectory(prefix="dashboard-demo-") as tmp:
+        paths = build_host(Path(tmp), args.hotfix)
+        units.run = FakeSystemd(paths.product, {n: s for n, s in INSTANCES.items() if s})
+        print(f"pretend host in {tmp}; {FLAKY} crashes every {CRASH_EVERY:.0f}s, "
+              f"{REFUSED} refuses every action", flush=True)
+        try:
+            return dashboard.dashboard(paths, args)
+        except KeyboardInterrupt:
+            return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/instrumentctl/cli.py

@@ -10,6 +10,7 @@
 from typing import List, Optional
 
 from . import commands, hotfix
+from .dashboard import dashboard
 from .doctor import doctor
 from .installer import install
 from .instances import Paths
@@ -35,6 +36,7 @@ def _epilog(product: Product) -> str:
         (f"restart {instance}", f"e.g. after editing {etc}/{instance}.yaml"),
         (f"logs {instance} -f", "follow its log"),
         ("doctor", "why isn't it working?"),
+        ("dashboard", "serve a web page of this host's daemons, with start and stop"),
         ('hotfix "what you fixed"', "save an edit to the code here and send it for review"),
         ("update", "back to the reviewed code, once the hotfix is merged"),
     ]
@@ -58,6 +60,20 @@ def _add_undeploy(add, product: Product) -> None:
     p.add_argument("-n", "--dry-run", action="store_true", help="print what would happen")
 
 
+def _add_dashboard(add, product: Product) -> None:
+    p = add("dashboard", dashboard,
+            "serve a web page showing this host's daemons, with start and stop",
+            "Serve a page showing every daemon deployed here, refreshed every two\n"
+            "seconds, with start, stop and restart. There is no login: anyone who can\n"
+            "reach it can stop the daemons, so it listens on loopback unless\n"
+            "[dashboard] bind in instrument.toml says otherwise. Normally run by\n"
+            f"{product.name}-dashboard.service rather than by hand.")
+    p.add_argument("--bind", metavar="ADDRESS",
+                   help=f"address to listen on (default {product.dashboard_bind})")
+    p.add_argument("--port", type=int,
+                   help=f"port to listen on (default {product.dashboard_port})")
+
+
 def _parser(product: Product) -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         prog=product.name,
@@ -156,9 +172,11 @@ def add(name, func, help_text, description=None):
 
     add("doctor", doctor, "check this host and your account, and say what to fix")
 
+    _add_dashboard(add, product)
+
     p = add("install", install, "set this host up (root)",
             "Create the ops group and the daemon account, enrol operators, make the\n"
-            "directories, and render the unit, polkit rule, sudoers drop-in and helper\n"
+            "directories, and render the units, polkit rule, sudoers drop-in and helper\n"
             "scripts. Safe to re-run. The virtualenv must already exist, since this\n"
             "code lives in it.")
     p.add_argument("operators", nargs="*", metavar="user",

src/instrumentctl/commands.py

@@ -255,11 +255,9 @@ def _act(product: Product, names: List[str], verb: str, dry_run: bool) -> int:
     state = units.states(product, names)
     failed = []
     for name in names:
-        if verb == "start" and state[name].active == "active":
-            print(f"{name:<24} already running")
-            continue
-        if verb == "stop" and not state[name].running:
-            print(f"{name:<24} not running")
+        skip = units.needless(state[name], verb)
+        if skip:
+            print(f"{name:<24} {skip}")
             continue
         if dry_run:
             print(f"{name:<24} would {verb}")