feat: add update metrics + tests

.github/workflows/github-publish.yaml

@@ -16,8 +16,16 @@ on:
         type: string
 
 jobs:
+  smoke:
+    name: Smoke test
+    if: startsWith(github.ref, 'refs/tags/v') && !inputs.skip_publish
+    permissions:
+      contents: read
+    uses: ./.github/workflows/smoke-test.yaml
+
   publish:
     name: Push image to GHCR
+    needs: smoke
     runs-on: ubuntu-latest
     if: startsWith(github.ref, 'refs/tags/v') && !inputs.skip_publish
     timeout-minutes: 30

.github/workflows/smoke-test.yaml

@@ -0,0 +1,24 @@
+name: Smoke test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+  # called by github-publish.yaml before pushing a release
+  workflow_call:
+
+jobs:
+  smoke:
+    name: Image starts and serves metrics
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Run smoke test
+        run: tests/smoke.sh

Dockerfile

@@ -5,10 +5,22 @@ FROM quay.io/prometheus/node-exporter:v1.12.1
 # It listens on LISTEN_ADDRESS, by default the gateway of the paashup traefik network
 # (172.31.0.0/16): Prometheus reaches it there, and the port is not exposed on the public IP.
 #   - udev data: disk model/serial in node_disk_info
-#   - textfile: node_reboot_required / apt_upgrades_pending, written on the host by the
-#     node-exporter-apt timer (see paashup-install.sh) into /var/lib/node_exporter/textfile
-ENV LISTEN_ADDRESS=172.31.0.1:9100
+#   - textfile: node_reboot_required, apt_upgrades_pending and the unattended-upgrades reboot
+#     settings, written by apt-metrics.sh (host apt, through chroot /host) every
+#     APT_METRICS_INTERVAL seconds
+ENV LISTEN_ADDRESS=172.31.0.1:9100 \
+    APT_METRICS_INTERVAL=300
 
+# root only for apt-metrics.sh: chroot /host needs it to run the host's apt (the host stays
+# mounted read-only). node_exporter itself, the part on the network, runs as nobody.
+USER root
+
+# apt needs a writable /tmp in the chroot when its binary cache is missing or stale (it then
+# re-reads the InRelease files). A volume there, not the host's /tmp: the rest of /host stays
+# read-only and nothing has to be added at run time.
+VOLUME /host/tmp
+
+COPY --chmod=755 apt-metrics.sh /apt-metrics.sh
 COPY --chmod=755 entrypoint.sh /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]
 CMD []

README.md

@@ -3,7 +3,8 @@
 Repackages `quay.io/prometheus/node-exporter` as a versioned, buildable
 image. It reports the VM itself: OS, kernel, uptime,
 CPU/RAM/disk/network, and through the textfile collector the pending
-reboot (`node_reboot_required`) and upgrades (`apt_upgrades_pending`).
+reboot (`node_reboot_required`), the upgrades (`apt_upgrades_pending`) and
+the unattended-upgrades reboot settings.
 
 - `Dockerfile` → static metadata (Prometheus scrape labels) and default
   flags (`--path.rootfs=/host`, udev data, textfile directory).
@@ -32,9 +33,26 @@ is not exposed on the public IP. If that network has another subnet, set
 `prometheus_address=<gateway>:9100`. Without the `traefik` network the
 container exits (`cannot assign requested address`).
 
-For the textfile metrics, the `node-exporter-apt` timer installed by
-`paashup-install.sh` writes `/var/lib/node_exporter/textfile/apt.prom`
-on the host. Without it these two metrics are simply absent.
+The textfile metrics are written by `apt-metrics.sh`, started next to
+node_exporter: every `APT_METRICS_INTERVAL` seconds (default 300) it runs
+the host's apt through `chroot /host` and writes `/tmp/textfile/apt.prom`
+inside the container. Nothing has to be installed on the host.
+When apt's binary cache is missing or stale, apt needs a writable `/tmp`
+in the chroot: the image declares a volume on `/host/tmp` for it (the
+host's `/tmp` is not touched, the rest of `/host` stays read-only).
+`chroot` needs root, so only this loop runs as root (it reads the host,
+mounted read-only, and listens on nothing); node_exporter, the part on
+the network, runs as `nobody`.
+
+| Metric | Value |
+|---|---|
+| `node_reboot_required` | 1 if `/run/reboot-required` exists |
+| `apt_upgrades_pending` | packages `apt-get -s upgrade` would install |
+| `apt_unattended_upgrades_enabled` | 1 if `APT::Periodic::Unattended-Upgrade` is set |
+| `apt_auto_reboot_enabled` | `Unattended-Upgrade::Automatic-Reboot` |
+| `apt_auto_reboot_time_seconds{time,timezone}` | `Automatic-Reboot-Time`, in seconds after midnight in the host time zone (absent when it is `now`) |
+| `apt_auto_reboot_timezone_offset_seconds{timezone}` | UTC offset of the host time zone |
+| `node_reboot_scheduled_timestamp_seconds` | time of the reboot planned by unattended-upgrades, 0 if none |
 
 ## Usage
 
@@ -58,6 +76,21 @@ docker run -d \
 
 Metrics are exposed on `http://172.31.0.1:9100/metrics`.
 
+## Tests
+
+```bash
+tests/smoke.sh                 # builds the image, then tests it
+IMAGE=<ref> tests/smoke.sh     # tests an existing image
+```
+
+It runs the container as documented (on `127.0.0.1:$SMOKE_PORT`,
+default 19100) and checks the labels, the core collectors, `/` seen
+through `--path.rootfs`, node_exporter running as `nobody`, run-time
+arguments added to the baked flags, and the apt metrics, against both
+the real host and a Debian rootfs with a known unattended-upgrades
+config. CI runs it on every push to `main`, on pull requests and
+before each release.
+
 ## Release
 
 Push a `vX.Y.Z` tag: the workflow builds and pushes

apt-metrics.sh

@@ -0,0 +1,91 @@
+#!/bin/sh
+# Pending reboot and upgrades of the host, and its unattended-upgrades reboot settings, read
+# through /host, for the textfile collector. Started in the background by entrypoint.sh and
+# rewritten every APT_METRICS_INTERVAL seconds. apt runs from the host (chroot /host), read-only.
+dir="$1"
+interval="${APT_METRICS_INTERVAL:-300}"
+
+# value of an apt option, as `apt-config dump` prints it: Key "value";
+conf_get() {
+  printf '%s\n' "$conf" | sed -n "s|^$1 \"\(.*\)\";$|\1|p" | tail -n 1
+}
+
+is_true() {
+  case "$1" in true|yes|on|1) return 0 ;; *) return 1 ;; esac
+}
+
+write_metrics() {
+  required=0
+  [ -f /host/run/reboot-required ] && required=1
+
+  pending=""
+  if sim="$(chroot /host apt-get -s -o Debug::NoLocking=1 upgrade 2>&1)"; then
+    pending="$(printf '%s\n' "$sim" | grep -c '^Inst ')"
+  else
+    echo "apt-metrics: apt-get -s upgrade failed: $(printf '%s\n' "$sim" | grep '^E:' | head -n 2)" >&2
+  fi
+
+  conf="$(chroot /host apt-config dump 2>/dev/null)"
+  periodic="$(conf_get 'APT::Periodic::Unattended-Upgrade')"
+  auto="$(conf_get 'Unattended-Upgrade::Automatic-Reboot')"
+  at="$(conf_get 'Unattended-Upgrade::Automatic-Reboot-Time')"
+  [ -n "$at" ] || at="now"
+
+  enabled=0
+  [ -n "$periodic" ] && [ "$periodic" != 0 ] && enabled=1
+  auto_reboot=0
+  is_true "$auto" && auto_reboot=1
+
+  tz="$(cat /host/etc/timezone 2>/dev/null || readlink /host/etc/localtime | sed 's|.*/zoneinfo/||')"
+  [ -n "$tz" ] || tz="UTC"
+  # +0200 -> 7200
+  offset="$(chroot /host date +%z 2>/dev/null)"
+  case "$offset" in
+    [+-][0-9][0-9][0-9][0-9])
+      sign="${offset%????}"; hh="${offset#?}"; hh="${hh%??}"; mm="${offset#???}"
+      offset=$(( ${sign}1 * (${hh#0} * 3600 + ${mm#0} * 60) )) ;;
+    *) offset=0 ;;
+  esac
+
+  # reboot planned by unattended-upgrades (shutdown -r HH:MM), in microseconds since epoch
+  scheduled=0
+  if grep -q '^MODE=reboot' /host/run/systemd/shutdown/scheduled 2>/dev/null; then
+    usec="$(sed -n 's/^USEC=//p' /host/run/systemd/shutdown/scheduled)"
+    [ -n "$usec" ] && scheduled=$(( usec / 1000000 ))
+  fi
+
+  {
+    echo '# HELP node_reboot_required 1 if a reboot is pending.'
+    echo '# TYPE node_reboot_required gauge'
+    echo "node_reboot_required $required"
+    if [ -n "$pending" ]; then
+      echo '# HELP apt_upgrades_pending Packages waiting to be upgraded.'
+      echo '# TYPE apt_upgrades_pending gauge'
+      echo "apt_upgrades_pending $pending"
+    fi
+    echo '# HELP apt_unattended_upgrades_enabled 1 if unattended-upgrades runs every day.'
+    echo '# TYPE apt_unattended_upgrades_enabled gauge'
+    echo "apt_unattended_upgrades_enabled $enabled"
+    echo '# HELP apt_auto_reboot_enabled 1 if unattended-upgrades reboots by itself when an update needs it.'
+    echo '# TYPE apt_auto_reboot_enabled gauge'
+    echo "apt_auto_reboot_enabled $auto_reboot"
+    case "$at" in
+      [0-9]:[0-9][0-9]|[0-9][0-9]:[0-9][0-9])
+        h="${at%%:*}"; m="${at#*:}"
+        echo '# HELP apt_auto_reboot_time_seconds Configured reboot time, in seconds after midnight in the host time zone.'
+        echo '# TYPE apt_auto_reboot_time_seconds gauge'
+        echo "apt_auto_reboot_time_seconds{time=\"$at\",timezone=\"$tz\"} $(( ${h#0} * 3600 + ${m#0} * 60 ))" ;;
+    esac
+    echo '# HELP apt_auto_reboot_timezone_offset_seconds UTC offset of the host time zone, now.'
+    echo '# TYPE apt_auto_reboot_timezone_offset_seconds gauge'
+    echo "apt_auto_reboot_timezone_offset_seconds{timezone=\"$tz\"} $offset"
+    echo '# HELP node_reboot_scheduled_timestamp_seconds Time of the planned reboot (0 if none).'
+    echo '# TYPE node_reboot_scheduled_timestamp_seconds gauge'
+    echo "node_reboot_scheduled_timestamp_seconds $scheduled"
+  } > "$dir/apt.prom.$$" && mv "$dir/apt.prom.$$" "$dir/apt.prom"
+}
+
+while :; do
+  write_metrics || echo "apt-metrics: could not write $dir/apt.prom" >&2
+  sleep "$interval"
+done

entrypoint.sh

@@ -3,9 +3,12 @@
 # the host PID namespace and / mounted read-only at /host. Arguments given at run time are
 # added after these (they don't replace them).
 set -e
-exec /bin/node_exporter \
+TEXTFILE_DIR=/tmp/textfile
+mkdir -p "$TEXTFILE_DIR"
+/apt-metrics.sh "$TEXTFILE_DIR" &
+exec setuidgid nobody /bin/node_exporter \
   --path.rootfs=/host \
   --path.udev.data=/host/run/udev/data \
-  --collector.textfile.directory=/host/var/lib/node_exporter/textfile \
+  --collector.textfile.directory="$TEXTFILE_DIR" \
   --web.listen-address="${LISTEN_ADDRESS:-172.31.0.1:9100}" \
   "$@"

tests/smoke.sh

@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+# Smoke test: the image starts with the flags baked in entrypoint.sh, serves metrics, keeps the
+# run-time arguments, and apt-metrics.sh writes the textfile metrics from the host's apt.
+#   tests/smoke.sh               builds the image from the repo
+#   IMAGE=<ref> tests/smoke.sh   tests an existing image
+# Needs docker and curl. Listens on 127.0.0.1:$SMOKE_PORT (default 19100) with the host network.
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+
+BUILD=0
+if [ -z "${IMAGE:-}" ]; then IMAGE=node-exporter:smoke; BUILD=1; fi
+HOST_IMAGE=node-exporter-smoke-host
+PORT="${SMOKE_PORT:-19100}"
+ADDR="127.0.0.1:$PORT"
+NAME="node-exporter-smoke-$$"
+TMP="$(mktemp -d)"
+failures=0
+
+cleanup() {
+  docker rm -fv "$NAME" "$NAME-host" >/dev/null 2>&1 || true
+  rm -rf "$TMP"
+}
+trap cleanup EXIT
+
+pass() { echo "  ok   $*"; }
+fail() { echo "  FAIL $*"; failures=$((failures + 1)); }
+
+# expect <file> <regex> <label>
+expect() {
+  if grep -Eq "$2" "$1"; then pass "$3"; else fail "$3 (no line matching: $2)"; fi
+}
+
+# wait_metrics <path> <out> <regex>: retries until the endpoint serves a line matching regex
+# (apt-metrics.sh writes its file a few seconds after the start) or the container stops
+wait_metrics() {
+  for _ in $(seq 1 60); do
+    if curl -fsS "http://$ADDR$1" -o "$2" 2>/dev/null && grep -Eq "$3" "$2"; then return 0; fi
+    if [ "$(docker inspect -f '{{.State.Running}}' "$NAME" 2>/dev/null)" != "true" ]; then break; fi
+    sleep 1
+  done
+  echo "--- container logs ---"; docker logs "$NAME" 2>&1 | tail -30
+  return 1
+}
+
+if [ "$BUILD" = 1 ]; then
+  echo "== build $IMAGE"
+  docker build -q -t "$IMAGE" . >/dev/null
+fi
+
+echo "== labels and env"
+labels="$(docker image inspect -f '{{json .Config.Labels}}' "$IMAGE")"
+[[ "$labels" == *'"prometheus_scrape":"true"'* ]] && pass "prometheus_scrape=true" || fail "prometheus_scrape label: $labels"
+[[ "$labels" == *'"prometheus_address":"172.31.0.1:9100"'* ]] && pass "prometheus_address=172.31.0.1:9100" || fail "prometheus_address label: $labels"
+env="$(docker image inspect -f '{{json .Config.Env}}' "$IMAGE")"
+[[ "$env" == *'LISTEN_ADDRESS=172.31.0.1:9100'* ]] && pass "default LISTEN_ADDRESS" || fail "LISTEN_ADDRESS env: $env"
+
+echo "== default address without the traefik network"
+if ip -4 addr 2>/dev/null | grep -q 'inet 172\.31\.0\.1/'; then
+  echo "  skip 172.31.0.1 exists on this host"
+else
+  docker run -d --name "$NAME" --network host --pid host -v /:/host:ro,rslave "$IMAGE" >/dev/null
+  docker wait "$NAME" >/dev/null 2>&1 || true
+  if docker logs "$NAME" 2>&1 | grep -q 'cannot assign requested address'; then
+    pass "exits with 'cannot assign requested address'"
+  else
+    fail "expected exit with 'cannot assign requested address'"; docker logs "$NAME" 2>&1 | tail -10
+  fi
+  docker rm -fv "$NAME" >/dev/null
+fi
+
+echo "== run as documented (host /, host network, host PID)"
+docker run -d --name "$NAME" --network host --pid host -v /:/host:ro,rslave \
+  -e LISTEN_ADDRESS="$ADDR" "$IMAGE" >/dev/null
+m="$TMP/metrics"
+if wait_metrics /metrics "$m" '^node_reboot_required '; then
+  pass "metrics served on $ADDR"
+  expect "$m" '^node_exporter_build_info\{' "node_exporter_build_info"
+  expect "$m" '^node_uname_info\{' "node_uname_info"
+  expect "$m" '^node_boot_time_seconds ' "node_boot_time_seconds"
+  expect "$m" '^node_memory_MemTotal_bytes ' "node_memory_MemTotal_bytes"
+  # rootfs is /host: the host's / must be reported as mountpoint="/", not "/host"
+  expect "$m" '^node_filesystem_size_bytes\{[^}]*mountpoint="/"' "filesystem / seen through --path.rootfs"
+  if grep -Eq '^node_filesystem_size_bytes\{[^}]*mountpoint="/host' "$m"; then
+    fail "filesystem mountpoints still prefixed with /host"
+  else
+    pass "no /host-prefixed mountpoints"
+  fi
+  # collectors that must work on any VM (the others depend on hardware: zfs, nfs, infiniband...)
+  for c in cpu diskstats filesystem loadavg meminfo netdev stat time uname textfile; do
+    expect "$m" "^node_scrape_collector_success\{collector=\"$c\"\} 1$" "collector $c"
+  done
+  expect "$m" '^node_reboot_required [01]$' "node_reboot_required from the host"
+  if [ -x /usr/bin/apt-get ]; then
+    expect "$m" '^apt_upgrades_pending [0-9]+$' "apt_upgrades_pending from the host's apt"
+  else
+    echo "  skip apt_upgrades_pending: no apt on this host"
+  fi
+  expect "$m" '^node_textfile_scrape_error 0$' "node_textfile_scrape_error 0"
+else
+  fail "metrics not served on $ADDR"
+fi
+# default columns: `docker top -o` fails with --pid host
+procs="$(docker top "$NAME" 2>/dev/null || true)"
+if grep -E '/bin/node_exporter' <<<"$procs" | grep -Eq '^(nobody|65534) '; then
+  pass "node_exporter runs as nobody"
+else
+  fail "node_exporter not running as nobody: $procs"
+fi
+grep -q 'apt-metrics.sh' <<<"$procs" && pass "apt-metrics.sh running" || fail "apt-metrics.sh not running: $procs"
+if docker logs "$NAME" 2>&1 | grep -E 'level=ERROR|apt-metrics:'; then
+  fail "errors in container logs"
+else
+  pass "no error in logs"
+fi
+docker rm -fv "$NAME" >/dev/null
+
+echo "== apt metrics on a known host, run-time arguments"
+# a Debian host where a reboot is required, unattended-upgrades reboots at 02:30 Paris time
+# and has planned one; its package lists are there but not apt's binary cache, so apt has to
+# re-read them (and needs a writable /tmp)
+docker build -q -t "$HOST_IMAGE" - >/dev/null <<'EOF'
+FROM debian:trixie-slim
+RUN rm /etc/apt/apt.conf.d/docker-clean && apt-get update -qq && rm -f /var/cache/apt/*.bin
+RUN printf 'APT::Periodic::Update-Package-Lists "1";\nAPT::Periodic::Unattended-Upgrade "1";\n' \
+      > /etc/apt/apt.conf.d/20auto-upgrades \
+ && printf 'Unattended-Upgrade::Automatic-Reboot "true";\nUnattended-Upgrade::Automatic-Reboot-Time "02:30";\n' \
+      > /etc/apt/apt.conf.d/52reboot \
+ && ln -sf /usr/share/zoneinfo/Europe/Paris /etc/localtime && echo Europe/Paris > /etc/timezone \
+ && mkdir -p /run/systemd/shutdown && touch /run/reboot-required \
+ && printf 'USEC=1893461400000000\nWARN_WALL=1\nMODE=reboot\n' > /run/systemd/shutdown/scheduled
+EOF
+mkdir "$TMP/root"
+docker create --name "$NAME-host" "$HOST_IMAGE" >/dev/null
+docker export "$NAME-host" | tar -x -C "$TMP/root" --exclude='dev/*' 2>/dev/null || true
+docker rm "$NAME-host" >/dev/null
+# an extra argument must be added to the baked flags, not replace them
+docker run -d --name "$NAME" --network host --pid host -v "$TMP/root:/host:ro" \
+  -e LISTEN_ADDRESS="$ADDR" "$IMAGE" --web.telemetry-path=/smoke-metrics >/dev/null
+m="$TMP/metrics-host"
+if wait_metrics /smoke-metrics "$m" '^apt_upgrades_pending '; then
+  pass "run-time argument applied (--web.telemetry-path)"
+  expect "$m" '^node_reboot_required 1$' "node_reboot_required 1"
+  expect "$m" '^apt_upgrades_pending [0-9]+$' "apt_upgrades_pending without apt's binary cache"
+  expect "$m" '^apt_unattended_upgrades_enabled 1$' "apt_unattended_upgrades_enabled 1"
+  expect "$m" '^apt_auto_reboot_enabled 1$' "apt_auto_reboot_enabled 1"
+  expect "$m" '^apt_auto_reboot_time_seconds\{time="02:30",timezone="Europe/Paris"\} 9000$' "apt_auto_reboot_time_seconds 02:30"
+  expect "$m" '^apt_auto_reboot_timezone_offset_seconds\{timezone="Europe/Paris"\} (3600|7200)$' "Europe/Paris offset"
+  # node_exporter re-exposes textfile values as floats: 1893461400 -> 1.8934614e+09
+  expect "$m" '^node_reboot_scheduled_timestamp_seconds (1893461400|1\.8934614e\+09)$' "node_reboot_scheduled_timestamp_seconds"
+else
+  fail "apt metrics not served on $ADDR/smoke-metrics"
+fi
+
+echo
+if [ "$failures" -gt 0 ]; then
+  echo "$failures check(s) failed"
+  exit 1
+fi
+echo "all checks passed"