diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml index 90e5b90..352774d 100644 --- a/.github/workflows/pylint.yml +++ b/.github/workflows/pylint.yml @@ -23,7 +23,7 @@ jobs: - name: Install run: | python -m pip install -U pip - python -m pip install pylint . + python -m pip install pylint ".[dashboard]" - name: Run pylint run: pylint src/instrumentctl/ diff --git a/README.md b/README.md index 1bbfd4c..777b5ee 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,8 @@ common keys with their defaults, ready to copy. | `git.email_domain` | the hostname | operators commit as `@` | | `legacy.unit` | none | a template unit an older version installed, reported by `doctor` | | `legacy.polkit_rule` | none | likewise for its polkit rule | +| `dashboard.bind` | `127.0.0.1` | where the dashboard listens; `0.0.0.0` opens it to the network | +| `dashboard.port` | `8750` | likewise | Which credentials a host needs is deliberately not configured. It is derived from the deployed configs and from the daemon code that reads them, so it @@ -116,6 +118,29 @@ reviewed pull request. Fine-grained tokens expire, at most a year out, and `doctor` cannot tell an expired one from a good one, so put the date in a calendar. +## Dashboard + +` dashboard` serves a page showing every daemon deployed on the host: +state, restarts, PID, memory, whether the config was edited there, and whether +the checkout is on a hotfix. It refreshes every two seconds, and has start, +stop and restart for each instance and each subsystem, which act exactly as +the CLI's do (a subsystem stops in reverse order). + +It needs Flask and waitress, the `dashboard` extra: have the instrument depend +on `instrumentctl[dashboard]`. `install` renders `-dashboard.service`, +which runs it as the daemon account; polkit lets that account, as a member of +the ops group, start and stop the instrument's units and nothing else. +Whether a host serves it is left to an admin: + +```bash +sudo systemctl enable --now -dashboard +``` + +There is no login. Anyone who can reach the page can stop the daemons, so it +listens on loopback unless `[dashboard] bind` says otherwise, and that should +only be on a private network. Each action is logged with the address it came +from: `journalctl -u -dashboard`. + ## Tests ```bash @@ -126,6 +151,15 @@ They build a throwaway host tree and exercise real directory reads, stubbing only systemd. The rendering tests assert that no `@placeholder@` survives, which is what catches a template gaining a variable nobody wired up. +To see the dashboard without systemd, run it against a pretend host whose +daemons start, stop, crash and restart in memory: + +```bash +uv run --extra test python scripts/dashboard_demo.py # then open http://127.0.0.1:8750/ +``` + +`--hotfix` puts the pretend checkout on a hotfix branch, to see those warnings. + ## Host setup Every instrument installs the same way. The instrument's own docs add only its diff --git a/instrument.toml.example b/instrument.toml.example index cbd185b..2d173bc 100644 --- a/instrument.toml.example +++ b/instrument.toml.example @@ -30,3 +30,9 @@ name = "demo" # base = "main" # the branch hosts run and hotfixes are reviewed into # branch_prefix = "hotfix/" # email_domain = "" # operators commit as @; default: the hostname + +# ` dashboard`, served by -dashboard.service. It has no login and +# can stop every daemon, so it listens on loopback unless opened up here +[dashboard] +# bind = "127.0.0.1" # "0.0.0.0" for every interface, on a private network +# port = 8750 diff --git a/pyproject.toml b/pyproject.toml index 1ef8910..46b634a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,8 @@ dependencies = [ ] [project.optional-dependencies] -test = ["pytest"] +dashboard = ["flask>=3.0", "waitress>=3.0"] +test = ["pytest", "flask>=3.0", "waitress>=3.0"] [tool.setuptools.packages.find] where = ["src"] diff --git a/scripts/dashboard_demo.py b/scripts/dashboard_demo.py new file mode 100644 index 0000000..51b538a --- /dev/null +++ b/scripts/dashboard_demo.py @@ -0,0 +1,211 @@ +"""Run the dashboard against a pretend host, to see it without systemd. + + uv run --extra test python scripts/dashboard_demo.py [--port 8750] [--hotfix] + +Builds a throwaway host tree (repo checkout, /etc, instance files) in a +temporary directory and replaces systemctl with an in-memory one, so start, +stop and restart work and take a moment, as real daemons do. One instance +crashes every 30 seconds and is restarted, to show what a flapping daemon +looks like; one starts out failed; one is refused, as polkit would refuse an +account outside the ops group. +""" +from __future__ import annotations + +import argparse +import itertools +import subprocess +import sys +import tempfile +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional, Sequence + +from instrumentctl import dashboard, units +from instrumentctl.instances import Paths +from instrumentctl.product import Product + +INSTRUMENT_TOML = """\ +[instrument] +name = "demo" +display_name = "Demo Spectrograph" +""" + +# name -> state at start; the last is in the repo for this host, not deployed +INSTANCES = { + "power_pdu1": "active", + "power_pdu2": "active", + "fei_adc": "active", + "fei_atc": "active", + "fei_fw": "inactive", + "cam_ccd": "failed", + "guider_tt": None, +} +FLAKY = "fei_atc" # crashes every CRASH_EVERY seconds, then restarts +REFUSED = "power_pdu2" # every action on it is denied +CRASH_EVERY = 30.0 +TRANSITION = 1.5 # seconds a start or stop takes +RESTART_DELAY = 5.0 # RestartSec= in the real unit +pids = itertools.count(41000) + + +# One attribute per property the real systemctl show reports +@dataclass +class Unit: # pylint: disable=too-many-instance-attributes + """One pretend daemon.""" + + active: str + since: float + sub: str = "dead" + restarts: int = 0 + result: str = "success" + exit_status: int = 0 + pid: int = 0 + # (state, when): where a start or stop in progress will land + pending: Optional[tuple] = None + + def become(self, active: str, now: float) -> None: + """Change state, as systemd records it.""" + self.active, self.since, self.pending = active, now, None + self.sub = {"active": "running", "failed": "failed"}.get(active, "dead") + self.pid = next(pids) if active == "active" else 0 + + +class FakeSystemd: + """Stands in for ``units.run``: answers ``show`` and acts on start/stop/restart.""" + + def __init__(self, product: Product, initial: Dict[str, str]) -> None: + self.product = product + self.lock = threading.Lock() + now = time.time() + self.units = {name: Unit(active="inactive", since=now) + for name in initial} + for name, state in initial.items(): + self.units[name].become(state, now) + failed = self.units.get("cam_ccd") + if failed: + failed.result, failed.exit_status = "exit-code", 1 + + def __call__(self, cmd: Sequence[str], capture: bool = False, + prog: str = "") -> subprocess.CompletedProcess: + cmd = list(cmd) + with self.lock: + now = time.time() + for name, unit in self.units.items(): + self._settle(name, unit, now) + if cmd[:2] == ["systemctl", "show"]: + names = [self.product.instance_of(u) for u in cmd[4:]] + text = "\n\n".join(self._show(n) for n in names if n in self.units) + return subprocess.CompletedProcess(cmd, 0, text + "\n", "") + if cmd[:2] == ["systemctl", "--no-ask-password"]: + return self._act(cmd, cmd[2], self.product.instance_of(cmd[3]), now) + return subprocess.CompletedProcess(cmd, 1, "", f"demo: cannot fake {cmd}\n") + + @staticmethod + def _settle(name: str, unit: Unit, now: float) -> None: + if unit.pending and now >= unit.pending[1]: + unit.become(unit.pending[0], now) + if name == FLAKY and unit.active == "active" and now - unit.since >= CRASH_EVERY: + # Restart=on-failure: systemd waits, then tries again + unit.become("activating", now) + unit.sub, unit.result, unit.exit_status = "auto-restart", "exit-code", 1 + unit.restarts += 1 + unit.pending = ("active", now + RESTART_DELAY) + + def _act(self, cmd: List[str], verb: str, name: Optional[str], + now: float) -> subprocess.CompletedProcess: + if name not in self.units: + return subprocess.CompletedProcess(cmd, 5, "", f"Unit {cmd[3]} not found.\n") + if name == REFUSED: + return subprocess.CompletedProcess( + cmd, 1, "", f"Failed to {verb} {cmd[3]}: Access denied\n") + unit = self.units[name] + if verb == "stop": + unit.become("deactivating", now) + unit.pending = ("inactive", now + TRANSITION) + else: + unit.become("activating", now) + unit.sub, unit.result, unit.exit_status = "start", "success", 0 + unit.restarts = 0 + unit.pending = ("active", now + TRANSITION) + return subprocess.CompletedProcess(cmd, 0, "", "") + + def _show(self, name: str) -> str: + unit = self.units[name] + stamp = time.strftime("%a %Y-%m-%d %H:%M:%S %Z", time.localtime(unit.since)) + memory = str(30_000_000 + 1_000_000 * len(name)) if unit.pid else "[not set]" + return "\n".join([ + f"Id={self.product.unit(name)}", f"ActiveState={unit.active}", + f"SubState={unit.sub}", "UnitFileState=enabled", + f"StateChangeTimestamp={stamp}", f"NRestarts={unit.restarts}", + f"Result={unit.result}", f"ExecMainStatus={unit.exit_status}", + f"MainPID={unit.pid}", f"MemoryCurrent={memory}"]) + + +def _git(repo: Path, *args: str) -> None: + subprocess.run(["git", "-C", str(repo), "-c", "user.name=demo", + "-c", "user.email=demo@example.invalid", *args], + check=True, capture_output=True) + + +def build_host(root: Path, hotfix: bool) -> Paths: + """A checkout, an /etc with most instances deployed, and the instrument.toml.""" + repo, etc = root / "repo", root / "etc" + (repo / "systemd" / "instances").mkdir(parents=True) + (etc / "instances").mkdir(parents=True) + (etc / "host").write_text("demohost\n", encoding="utf-8") + for name, state in INSTANCES.items(): + subsystem, device = name.split("_", 1) + env = repo / "systemd" / "instances" / f"{name}.env" + env.write_text(f"DEMO_DAEMON={subsystem}/{device}_daemon.py\n" + "DEMO_HOST=demohost\n", encoding="utf-8") + config = repo / "config" / subsystem / f"{name}.yaml" + config.parent.mkdir(parents=True, exist_ok=True) + config.write_text("hardware:\n port: 9000\n", encoding="utf-8") + if state is not None: + (etc / "instances" / env.name).write_bytes(env.read_bytes()) + (etc / config.name).write_bytes(config.read_bytes()) + # Hardware values tuned on the host, as the dashboard should point out + (etc / "cam_ccd.yaml").write_text("hardware:\n port: 9001\n", encoding="utf-8") + + _git(repo, "init", "-q", "-b", "main") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", "Initial") + if hotfix: + _git(repo, "update-ref", "refs/remotes/origin/main", "HEAD") + _git(repo, "checkout", "-q", "-b", "hotfix/2026-10-27-demohost-adc-hangs") + (repo / "config" / "fei" / "fei_adc.yaml").write_text( + "hardware:\n port: 9002\n", encoding="utf-8") + _git(repo, "commit", "-q", "-am", "ADC hangs after power cycle") + (repo / "notes.txt").write_text("unsaved\n", encoding="utf-8") + + (root / "instrument.toml").write_text(INSTRUMENT_TOML, encoding="utf-8") + product = Product.load(root / "instrument.toml") + return Paths.from_env(product, environ={ + "DEMO_REPO_DIR": str(repo), "DEMO_ETC_DIR": str(etc), + "DEMO_VENV_DIR": str(root / "venv"), "DEMO_LOGS_DIR": str(root / "logs"), + }) + + +def main(argv: Optional[List[str]] = None) -> int: + """Build the pretend host and serve the dashboard until interrupted.""" + parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + parser.add_argument("--bind", default="127.0.0.1") + parser.add_argument("--port", type=int, default=8750) + parser.add_argument("--hotfix", action="store_true", + help="put the checkout on a hotfix branch with an unsaved edit") + args = parser.parse_args(argv) + with tempfile.TemporaryDirectory(prefix="dashboard-demo-") as tmp: + paths = build_host(Path(tmp), args.hotfix) + units.run = FakeSystemd(paths.product, {n: s for n, s in INSTANCES.items() if s}) + print(f"pretend host in {tmp}; {FLAKY} crashes every {CRASH_EVERY:.0f}s, " + f"{REFUSED} refuses every action", flush=True) + try: + return dashboard.dashboard(paths, args) + except KeyboardInterrupt: + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/instrumentctl/cli.py b/src/instrumentctl/cli.py index 6a88e77..ffa1791 100644 --- a/src/instrumentctl/cli.py +++ b/src/instrumentctl/cli.py @@ -10,6 +10,7 @@ from typing import List, Optional from . import commands, hotfix +from .dashboard import dashboard from .doctor import doctor from .installer import install from .instances import Paths @@ -35,6 +36,7 @@ def _epilog(product: Product) -> str: (f"restart {instance}", f"e.g. after editing {etc}/{instance}.yaml"), (f"logs {instance} -f", "follow its log"), ("doctor", "why isn't it working?"), + ("dashboard", "serve a web page of this host's daemons, with start and stop"), ('hotfix "what you fixed"', "save an edit to the code here and send it for review"), ("update", "back to the reviewed code, once the hotfix is merged"), ] @@ -58,6 +60,20 @@ def _add_undeploy(add, product: Product) -> None: p.add_argument("-n", "--dry-run", action="store_true", help="print what would happen") +def _add_dashboard(add, product: Product) -> None: + p = add("dashboard", dashboard, + "serve a web page showing this host's daemons, with start and stop", + "Serve a page showing every daemon deployed here, refreshed every two\n" + "seconds, with start, stop and restart. There is no login: anyone who can\n" + "reach it can stop the daemons, so it listens on loopback unless\n" + "[dashboard] bind in instrument.toml says otherwise. Normally run by\n" + f"{product.name}-dashboard.service rather than by hand.") + p.add_argument("--bind", metavar="ADDRESS", + help=f"address to listen on (default {product.dashboard_bind})") + p.add_argument("--port", type=int, + help=f"port to listen on (default {product.dashboard_port})") + + def _parser(product: Product) -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog=product.name, @@ -156,9 +172,11 @@ def add(name, func, help_text, description=None): add("doctor", doctor, "check this host and your account, and say what to fix") + _add_dashboard(add, product) + p = add("install", install, "set this host up (root)", "Create the ops group and the daemon account, enrol operators, make the\n" - "directories, and render the unit, polkit rule, sudoers drop-in and helper\n" + "directories, and render the units, polkit rule, sudoers drop-in and helper\n" "scripts. Safe to re-run. The virtualenv must already exist, since this\n" "code lives in it.") p.add_argument("operators", nargs="*", metavar="user", diff --git a/src/instrumentctl/commands.py b/src/instrumentctl/commands.py index 485e0d9..332c4fc 100644 --- a/src/instrumentctl/commands.py +++ b/src/instrumentctl/commands.py @@ -255,11 +255,9 @@ def _act(product: Product, names: List[str], verb: str, dry_run: bool) -> int: state = units.states(product, names) failed = [] for name in names: - if verb == "start" and state[name].active == "active": - print(f"{name:<24} already running") - continue - if verb == "stop" and not state[name].running: - print(f"{name:<24} not running") + skip = units.needless(state[name], verb) + if skip: + print(f"{name:<24} {skip}") continue if dry_run: print(f"{name:<24} would {verb}") diff --git a/src/instrumentctl/dashboard.py b/src/instrumentctl/dashboard.py new file mode 100644 index 0000000..e958b0d --- /dev/null +++ b/src/instrumentctl/dashboard.py @@ -0,0 +1,222 @@ +"""``dashboard``: a web page showing this host's daemons, with start, stop and restart. + +The three verbs go through systemctl just as they do from the CLI, so polkit +decides: it lets the ops group, and with it the daemon account the service +runs as, act on ``@`` units and nothing else. There is no login. + +The page asks for ``/api/status`` every couple of seconds. One ``systemctl +show`` answers for every instance and is shared between viewers for a second, +so ten open tabs cost no more than one. + +Flask and waitress are the ``dashboard`` extra and imported only here, so the +rest of the CLI keeps working on a host without them. +""" +from __future__ import annotations + +import argparse +import sys +import threading +import time +from typing import Any, Callable, Dict, List, Tuple + +from . import hotfix +from . import instances as inst +from . import units +from .installer import _render +from .instances import Paths, TargetError + +VERBS = ("start", "stop", "restart") +_DONE = {"start": "started", "stop": "stopped", "restart": "restarted"} + +# How long one answer is shared. The checkout changes far less often than the +# daemons, and asking git costs several processes. +STATUS_TTL = 1.0 +CHECKOUT_TTL = 30.0 + + +class Cached: + """A value recomputed at most every ``ttl`` seconds, however many ask.""" + + def __init__(self, ttl: float, compute: Callable[[], Any]) -> None: + self.ttl = ttl + self.compute = compute + self.lock = threading.Lock() + self.value: Any = None + self.taken = float("-inf") + + def get(self) -> Any: + """The value, recomputed first if it is older than ``ttl``.""" + with self.lock: + if time.monotonic() - self.taken >= self.ttl: + self.value = self.compute() + self.taken = time.monotonic() + return self.value + + def clear(self) -> None: + """Make the next ``get`` recompute, e.g. after starting something.""" + with self.lock: + self.taken = float("-inf") + + +def _config_state(paths: Paths, name: str) -> str: + """repo, differs, or missing: is the deployed config the committed one?""" + config = paths.config(name) + if not config.is_file(): + return "missing" + found = inst.repo_configs(paths, name) + if len(found) == 1 and config.read_bytes() != found[0].read_bytes(): + return "differs" + return "repo" + + +def _daemon(paths: Paths, name: str) -> str: + try: + env = inst.read_env(paths.instances / f"{name}.env") + except OSError: + return "?" + return env.get(paths.product.env_key("DAEMON"), "?") + + +def _instance(paths: Paths, name: str, state: units.UnitState) -> Dict[str, Any]: + return { + "name": name, + # What `start ` matches, so a group's buttons act on the group + "subsystem": name.split("_", 1)[0], + "daemon": _daemon(paths, name), + "config": _config_state(paths, name), + "active": state.active, + "sub": state.sub, + "boot": state.enabled, + "since": state.since if state.active != "inactive" else "", + "restarts": state.restarts, + "result": state.result, + "exit_status": state.exit_status, + "pid": state.pid, + "memory": state.memory, + } + + +def snapshot(paths: Paths) -> Dict[str, Any]: + """Every deployed instance and its state, plus what belongs here but is not.""" + product = paths.product + here = inst.deployed(paths) + role = inst.host_role(paths) + state = units.states(product, here) + # Another host's instances are its business, not something missing here + waiting = [name for name in inst.in_repo(paths) if name not in here + and inst.assigned_host(product, paths.repo_instances / f"{name}.env") + in (None, role)] + return { + "instrument": {"name": product.name, "display_name": product.display_name}, + "host": role, + "taken": time.time(), + "instances": [_instance(paths, name, state[name]) for name in here], + "undeployed": waiting, + } + + +def checkout(paths: Paths) -> Dict[str, Any]: + """Which code the daemons run: the branch, and anything a hotfix left behind.""" + problem = hotfix.checkout_problem(paths) + if problem: + return {"problem": problem} + branch = hotfix.current_branch(paths) + found = hotfix.changes(paths) + return { + "branch": branch, + "base": paths.product.git_base, + "hotfix": hotfix.is_hotfix(paths, branch), + "edited": len(found.edited) + len(found.new), + "unsent": hotfix.unsent(paths, branch) if branch else 0, + } + + +def act(paths: Paths, verb: str, targets: List[str], + origin: str) -> Tuple[int, Dict[str, Any]]: + """Apply a verb to instances or subsystems, as `` `` would. + + Returns an HTTP status and the per-instance outcome. + """ + product = paths.product + try: + names = inst.resolve(targets, inst.deployed(paths)) + except TargetError as exc: + return 404, {"error": f"not deployed on this host: {exc}"} + if verb == "stop": + # As from the CLI: undo a start in the opposite order + names.reverse() + state = units.states(product, names) + results = [] + for name in names: + skip = units.needless(state[name], verb) + if skip: + results.append({"name": name, "ok": True, "detail": skip}) + continue + # Fail rather than wait for a password nobody can type + proc = units.run(["systemctl", "--no-ask-password", verb, product.unit(name)], + capture=True) + ok = proc.returncode == 0 + detail = _DONE[verb] if ok else (proc.stderr.strip().splitlines() or ["failed"])[-1] + results.append({"name": name, "ok": ok, "detail": detail}) + print(f"{origin} {verb} {name}: {detail}", flush=True) + failed = any(not r["ok"] for r in results) + return (502 if failed else 200), {"results": results} + + +def create_app(paths: Paths) -> Any: + """The Flask application: the page, its status feed, and the three verbs.""" + # Deferred, so a host without the dashboard extra can still run the rest + from flask import Flask, Response, jsonify, request # pylint: disable=import-outside-toplevel + + app = Flask(paths.product.name) + page = _render("dashboard.html", paths.product, paths) + status = Cached(STATUS_TTL, lambda: snapshot(paths)) + code = Cached(CHECKOUT_TTL, lambda: checkout(paths)) + + @app.get("/") + def index() -> Response: + return Response(page, mimetype="text/html") + + @app.get("/api/status") + def api_status() -> Response: + return jsonify({**status.get(), "checkout": code.get()}) + + @app.post("/api/") + def api_act(verb: str) -> Tuple[Response, int]: + if verb not in VERBS: + return jsonify({"error": f"unknown action: {verb}"}), 404 + # A page elsewhere can make a browser on this network post a form + # here, but not a JSON body, which needs a CORS preflight this never + # answers + if not request.is_json: + return jsonify({"error": "send the targets as JSON"}), 415 + body = request.get_json(silent=True) + targets = body.get("targets") if isinstance(body, dict) else None + if (not isinstance(targets, list) or not targets + or not all(isinstance(t, str) for t in targets)): + return jsonify({"error": 'expected {"targets": ["", ...]}'}), 400 + http_status, body = act(paths, verb, targets, request.remote_addr or "?") + status.clear() + return jsonify(body), http_status + + return app + + +def dashboard(paths: Paths, args: argparse.Namespace) -> int: + """Serve the dashboard until interrupted.""" + product = paths.product + try: + import waitress # pylint: disable=import-outside-toplevel + app = create_app(paths) + except ImportError as exc: + print(f"{product.name}: the dashboard needs Flask and waitress ({exc.name} is " + f"missing); install instrumentctl[dashboard] in {paths.venv}", + file=sys.stderr) + return 1 + bind: str = args.bind or product.dashboard_bind + port: int = args.port or product.dashboard_port + print(f"{product.name} dashboard on http://{bind}:{port}/", flush=True) + # Each open page polls; a few threads keep one slow systemctl from + # stalling the rest + waitress.serve(app, host=bind, port=port, threads=8, ident=product.name) + return 0 diff --git a/src/instrumentctl/installer.py b/src/instrumentctl/installer.py index a4aaae0..24ae6a1 100644 --- a/src/instrumentctl/installer.py +++ b/src/instrumentctl/installer.py @@ -2,11 +2,11 @@ Run as root, and safe to re-run. It creates the ops group and the daemon account, enrols operators, makes the directories, and renders the unit, the -polkit rule, the sudoers drop-in, the two helper scripts and the login profile -from the same ``Product`` the rest of the CLI reads, so what ``doctor`` checks -is what this wrote. Last it gives ``/opt/`` and ``/etc/`` to the -ops group, all but ``secrets.env``, the GitHub token and the deploy key, -which stay root-owned. +polkit rule, the sudoers drop-in, the two helper scripts, the login profile and +the dashboard's unit from the same ``Product`` the rest of the CLI reads, so +what ``doctor`` checks is what this wrote. Last it gives ``/opt/`` and +``/etc/`` to the ops group, all but ``secrets.env``, the GitHub token and +the deploy key, which stay root-owned. Creating the virtualenv is not here. It has to happen before this runs, since this code lives inside it. @@ -159,7 +159,7 @@ def _secrets_and_role(product: Product, paths: Paths, role: Optional[str]) -> No def _artefacts(product: Product, paths: Paths) -> None: - """Render the unit, the polkit rule, the sudoers drop-in, helpers and profile.""" + """Render the units, the polkit rule, the sudoers drop-in, helpers and profile.""" _write(product.unit_template, _render("unit.service", product, paths), 0o644) product.polkit_rule.parent.mkdir(parents=True, exist_ok=True) product.polkit_rule.parent.chmod(0o750) @@ -169,6 +169,7 @@ def _artefacts(product: Product, paths: Paths) -> None: # can write to _write(product.enable_helper, _render("enable-helper.sh", product, paths), 0o755) _write(product.profile_script, _render("profile.sh", product, paths), 0o644) + _write(product.dashboard_unit, _render("dashboard.service", product, paths), 0o644) _sudoers(product, paths) @@ -331,6 +332,16 @@ def install(paths: Paths, args: argparse.Namespace) -> int: if pending: print(f"\nDeployed but not set to start at boot: {' '.join(pending)}") print(f" {product.name} enable --now {' '.join(pending)}") + # Likewise whether this host serves a page anyone on its network can use + unit = product.dashboard_unit.name + enabled = units.run(["systemctl", "is-enabled", unit], capture=True).stdout.strip() + if enabled != "enabled": + print(f"\nThe dashboard is not running; to serve it on " + f"{product.dashboard_bind}:{product.dashboard_port}:") + print(f" sudo systemctl enable --now {unit}") + else: + # It runs this code, which install may just have been run to update + subprocess.run(["systemctl", "try-restart", unit], check=False) return 0 diff --git a/src/instrumentctl/product.py b/src/instrumentctl/product.py index 749857e..faa3057 100644 --- a/src/instrumentctl/product.py +++ b/src/instrumentctl/product.py @@ -58,6 +58,10 @@ class Product: # pylint: disable=too-many-instance-attributes git_base: str = "main" git_branch_prefix: str = "hotfix/" git_email_domain: str = "" + # Where the dashboard listens. Loopback unless the instrument opens it up, + # since anyone who can reach it can start and stop the daemons + dashboard_bind: str = "127.0.0.1" + dashboard_port: int = 8750 @classmethod def load(cls, config: "ConfigSource") -> "Product": @@ -73,6 +77,7 @@ def load(cls, config: "ConfigSource") -> "Product": help_text = data.get("help", {}) paths = data.get("paths", {}) git = data.get("git", {}) + dashboard = data.get("dashboard", {}) return cls( name=instrument["name"], display_name=instrument.get("display_name", @@ -94,6 +99,8 @@ def load(cls, config: "ConfigSource") -> "Product": git_base=git.get("base", "main"), git_branch_prefix=git.get("branch_prefix", "hotfix/"), git_email_domain=git.get("email_domain", ""), + dashboard_bind=dashboard.get("bind", "127.0.0.1"), + dashboard_port=int(dashboard.get("port", 8750)), ) def unit(self, instance: str) -> str: @@ -116,6 +123,11 @@ def unit_template(self) -> Path: """The installed template unit every instance runs from.""" return SYSTEMD_DIR / f"{self.name}@.service" + @property + def dashboard_unit(self) -> Path: + """The service that serves the dashboard.""" + return SYSTEMD_DIR / f"{self.name}-dashboard.service" + @property def polkit_rule(self) -> Path: """The rule granting the ops group start/stop without a password.""" diff --git a/src/instrumentctl/templates/dashboard.html b/src/instrumentctl/templates/dashboard.html new file mode 100644 index 0000000..dceb7a5 --- /dev/null +++ b/src/instrumentctl/templates/dashboard.html @@ -0,0 +1,370 @@ + + + + + + +@display_name@ daemons + + + + +
+

@display_name@

+ + connecting… + +
+
+
+
+ + + + + + + + + +
StateInstanceSinceBootRestartsPIDMemoryDaemonConfig
Loading…
+
+
+ + + + diff --git a/src/instrumentctl/templates/dashboard.service b/src/instrumentctl/templates/dashboard.service new file mode 100644 index 0000000..2d4d17b --- /dev/null +++ b/src/instrumentctl/templates/dashboard.service @@ -0,0 +1,40 @@ +# Rendered by `@name@ install`. Edits here are lost on the next run; change the +# template in instrumentctl instead. +# +# Serves the @display_name@ dashboard: what runs on this host, with start, stop +# and restart. Where it listens is [dashboard] in instrument.toml. + +[Unit] +Description=@display_name@ dashboard +Documentation=@docs_url@ +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +# The daemon account is in @ops_group@, which the polkit rule lets start and +# stop @name@@ units and nothing else +User=@user@ +Group=@user@ +ExecStart=@venv@/bin/@name@ dashboard +Environment=PYTHONUNBUFFERED=1 +# Reading the checkout's git state must not try to write its index, which the +# sandboxing below makes read-only +Environment=GIT_OPTIONAL_LOCKS=0 + +Restart=on-failure +RestartSec=5 + +# Every start, stop and restart is logged with the address it came from: +# journalctl -u @name@-dashboard +StandardOutput=journal +StandardError=journal +SyslogIdentifier=@name@-dashboard + +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/src/instrumentctl/units.py b/src/instrumentctl/units.py index e9b26f6..e26d807 100644 --- a/src/instrumentctl/units.py +++ b/src/instrumentctl/units.py @@ -7,7 +7,7 @@ import subprocess import sys from dataclasses import dataclass -from typing import Dict, List, Sequence +from typing import Dict, List, Optional, Sequence from .product import Product @@ -24,13 +24,20 @@ def run(cmd: Sequence[str], capture: bool = False, return subprocess.CompletedProcess(list(cmd), 127, "", message + "\n") +# Each attribute is one property systemd reports @dataclass(frozen=True) -class UnitState: +class UnitState: # pylint: disable=too-many-instance-attributes """What systemd says about one instance.""" active: str # active, inactive, failed, activating, deactivating enabled: str # enabled, disabled, or empty if systemd does not know it since: str # when ``active`` last changed, as systemd prints it + sub: str = "" # running, exited, auto-restart, dead, ... + restarts: int = 0 # automatic restarts since the unit was last started + result: str = "" # success, or why it last stopped: exit-code, signal, ... + exit_status: Optional[int] = None + pid: Optional[int] = None + memory: Optional[int] = None # bytes, when accounting is on @property def running(self) -> bool: @@ -40,14 +47,16 @@ def running(self) -> bool: UNKNOWN = UnitState(active="unknown", enabled="", since="") +_PROPERTIES = ("Id", "ActiveState", "SubState", "UnitFileState", "StateChangeTimestamp", + "NRestarts", "Result", "ExecMainStatus", "MainPID", "MemoryCurrent") + def states(product: Product, names: List[str]) -> Dict[str, UnitState]: """Query every instance in one systemctl call.""" if not names: return {} proc = run( - ["systemctl", "show", "--no-pager", - "--property=Id,ActiveState,UnitFileState,StateChangeTimestamp", + ["systemctl", "show", "--no-pager", f"--property={','.join(_PROPERTIES)}", *[product.unit(n) for n in names]], capture=True, ) @@ -61,16 +70,40 @@ def states(product: Product, names: List[str]) -> Dict[str, UnitState]: active=props.get("ActiveState", "unknown"), enabled=props.get("UnitFileState", ""), since=_short_time(props.get("StateChangeTimestamp", "")), + sub=props.get("SubState", ""), + restarts=_number(props.get("NRestarts")) or 0, + result=props.get("Result", ""), + exit_status=_number(props.get("ExecMainStatus")), + # 0 means no process, not process 0 + pid=_number(props.get("MainPID")) or None, + memory=_number(props.get("MemoryCurrent")), ) return {n: found.get(n, UNKNOWN) for n in names} +def _number(value: Optional[str]) -> Optional[int]: + # systemd prints "[not set]", or 2**64 - 1, for a value it does not track + if not value or not value.isdigit(): + return None + number = int(value) + return None if number >= 2 ** 63 else number + + def _short_time(stamp: str) -> str: # "Mon 2026-09-21 09:14:02 HST" -> "2026-09-21 09:14:02" parts = stamp.split() return " ".join(parts[1:3]) if len(parts) >= 3 else stamp +def needless(state: UnitState, verb: str) -> Optional[str]: + """Why ``verb`` would do nothing to a unit in ``state``, or None if it would act.""" + if verb == "start" and state.active == "active": + return "already running" + if verb == "stop" and not state.running: + return "not running" + return None + + def systemctl(product: Product, verb: str, name: str) -> bool: """start/stop/restart one instance. Output goes to the terminal.""" return run(["systemctl", verb, product.unit(name)], diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py new file mode 100644 index 0000000..824a955 --- /dev/null +++ b/tests/test_dashboard.py @@ -0,0 +1,156 @@ +"""The dashboard: what the page is told, and what its buttons are allowed to do.""" +from __future__ import annotations + +import subprocess +from typing import Dict, List, Sequence + +import pytest + +from instrumentctl import dashboard, units +from instrumentctl.instances import Paths + + +class FakeSystemd: + """Answers ``systemctl show`` from a table, and records every other call.""" + + def __init__(self, active: Dict[str, str], refuse: Sequence[str] = ()) -> None: + self.active = active + self.refuse = refuse + self.calls: List[List[str]] = [] + + def __call__(self, cmd: Sequence[str], capture: bool = False, + prog: str = "") -> subprocess.CompletedProcess: + cmd = list(cmd) + if cmd[:2] == ["systemctl", "show"]: + blocks = [] + for unit in cmd[4:]: + name = unit.split("@", 1)[1].removesuffix(".service") + blocks.append("\n".join([ + f"Id={unit}", f"ActiveState={self.active.get(name, 'inactive')}", + "SubState=running", "UnitFileState=enabled", + "StateChangeTimestamp=Mon 2026-10-26 09:14:02 PDT", + "NRestarts=2", "Result=success", "ExecMainStatus=0", + "MainPID=4242", "MemoryCurrent=[not set]"])) + return subprocess.CompletedProcess(cmd, 0, "\n\n".join(blocks) + "\n", "") + self.calls.append(cmd) + if cmd[-1].split("@", 1)[1].removesuffix(".service") in self.refuse: + return subprocess.CompletedProcess(cmd, 1, "", "Failed: Access denied\n") + return subprocess.CompletedProcess(cmd, 0, "", "") + + +@pytest.fixture(name="systemd") +def systemd_fixture(monkeypatch) -> FakeSystemd: + fake = FakeSystemd({"red_adc": "active", "blue_fw": "failed"}) + monkeypatch.setattr(units, "run", fake) + return fake + + +@pytest.fixture(name="client") +def client_fixture(paths: Paths, systemd: FakeSystemd): + del systemd # stubbed for the app's sake + return dashboard.create_app(paths).test_client() + + +def test_the_snapshot_reports_every_deployed_instance(paths: Paths, + systemd: FakeSystemd) -> None: + del systemd + found = {i["name"]: i for i in dashboard.snapshot(paths)["instances"]} + assert set(found) == {"red_adc", "blue_fw"} + assert found["red_adc"]["active"] == "active" + assert found["red_adc"]["subsystem"] == "red" + assert found["red_adc"]["daemon"] == "arm/runner.py" + assert found["red_adc"]["restarts"] == 2 + assert found["red_adc"]["pid"] == 4242 + assert found["red_adc"]["since"] == "2026-10-26 09:14:02" + # Not tracked, rather than a number that means nothing + assert found["red_adc"]["memory"] is None + assert found["blue_fw"]["active"] == "failed" + + +def test_undeployed_lists_only_what_belongs_on_this_host(paths: Paths, + systemd: FakeSystemd) -> None: + del systemd + # elsewhere_arm is assigned to otherhost, so it is not missing here + assert dashboard.snapshot(paths)["undeployed"] == ["red_fw"] + + +def test_a_config_edited_on_the_host_is_flagged(paths: Paths, systemd: FakeSystemd) -> None: + del systemd + paths.config("red_adc").write_text("hardware:\n port: 9999\n", encoding="utf-8") + paths.config("blue_fw").unlink() + found = {i["name"]: i["config"] for i in dashboard.snapshot(paths)["instances"]} + assert found == {"red_adc": "differs", "blue_fw": "missing"} + + +def test_the_page_carries_the_instruments_own_name(client) -> None: + page = client.get("/").get_data(as_text=True) + assert "Demo daemons" in page + assert 'const PROG = "demo"' in page + + +def test_status_includes_the_checkout(client) -> None: + body = client.get("/api/status").get_json() + assert body["host"] == "thishost" + # The test repo is not a git checkout, which is reported rather than fatal + assert "not a git checkout" in body["checkout"]["problem"] + + +def test_a_subsystem_stops_in_reverse_order(client, systemd: FakeSystemd) -> None: + response = client.post("/api/stop", json={"targets": ["blue_fw", "red_adc"]}) + assert response.status_code == 200 + # blue_fw has failed, so is not running and needs no stop + assert [r["detail"] for r in response.get_json()["results"]] == ["stopped", "not running"] + assert systemd.calls == [["systemctl", "--no-ask-password", "stop", + "demo@red_adc.service"]] + + +def test_starting_what_runs_is_left_alone(client, systemd: FakeSystemd) -> None: + response = client.post("/api/start", json={"targets": ["red"]}) + assert response.get_json()["results"] == [ + {"name": "red_adc", "ok": True, "detail": "already running"}] + assert not systemd.calls + + +def test_a_refused_action_is_reported(client, systemd: FakeSystemd, capsys) -> None: + systemd.refuse = ["blue_fw"] + response = client.post("/api/restart", json={"targets": ["blue_fw"]}) + assert response.status_code == 502 + assert response.get_json()["results"][0] == { + "name": "blue_fw", "ok": False, "detail": "Failed: Access denied"} + # With no login, the journal is the only record of who did what + assert "restart blue_fw: Failed: Access denied" in capsys.readouterr().out + + +def test_only_deployed_instances_can_be_acted_on(client, systemd: FakeSystemd) -> None: + for target in ("red_fw", "sshd", "../etc"): + response = client.post("/api/start", json={"targets": [target]}) + assert response.status_code == 404 + assert not systemd.calls + + +def test_only_the_three_verbs_are_accepted(client, systemd: FakeSystemd) -> None: + for verb in ("enable", "kill", "mask"): + assert client.post(f"/api/{verb}", json={"targets": ["red_adc"]}).status_code == 404 + assert not systemd.calls + + +def test_a_form_post_from_another_page_is_refused(client, systemd: FakeSystemd) -> None: + """A form can be posted cross-site without a preflight; JSON cannot.""" + response = client.post("/api/stop", data={"targets": "red_adc"}) + assert response.status_code == 415 + assert not systemd.calls + + +@pytest.mark.parametrize("body", [{}, {"targets": []}, {"targets": "red_adc"}, + {"targets": [1]}, [], ["red_adc"]]) +def test_malformed_targets_are_refused(client, systemd: FakeSystemd, body) -> None: + assert client.post("/api/stop", json=body).status_code == 400 + assert not systemd.calls + + +def test_viewers_share_one_systemctl_call(monkeypatch) -> None: + computed = [] + clock = iter([0.0, 0.0, 0.5, 2.0, 2.0]) + monkeypatch.setattr(dashboard.time, "monotonic", lambda: next(clock)) + cached = dashboard.Cached(1.0, lambda: computed.append(1) or len(computed)) + assert [cached.get(), cached.get(), cached.get()] == [1, 1, 2] diff --git a/tests/test_render.py b/tests/test_render.py index 3da7aac..5bbbda4 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -11,7 +11,8 @@ from instrumentctl.product import Product TEMPLATES = ("unit.service", "polkit.rules", "sudoers", - "front-door.sh", "enable-helper.sh", "profile.sh") + "front-door.sh", "enable-helper.sh", "profile.sh", + "dashboard.service", "dashboard.html") PLACEHOLDER = re.compile(r"@[a-z_]+@") @@ -75,3 +76,11 @@ def test_the_profile_puts_the_venv_first_on_path_once(product: Product, paths: P env={"PATH": "/usr/bin:/bin"}, capture_output=True, text=True, check=True).stdout.strip() assert shown == f"{paths.venv}/bin:/usr/bin:/bin" + + +def test_the_dashboard_runs_as_the_account_polkit_trusts(product: Product, + paths: Paths) -> None: + """The daemon account is in the ops group, which may start and stop our units.""" + unit = _render("dashboard.service", product, paths) + assert "User=demouser" in unit + assert f"ExecStart={paths.venv}/bin/demo dashboard" in unit