From edb504bebcb0c05f90d748c5d4c0ca8f9a12e7ea Mon Sep 17 00:00:00 2001 From: parasol-ka Date: Fri, 23 Oct 2026 15:08:15 +0200 Subject: [PATCH 1/2] feat: add more detailed update metrics --- README.md | 4 ++++ apt-metrics.sh | 36 ++++++++++++++++++++++++++++++++++++ tests/smoke.sh | 20 +++++++++++++++++++- 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7bdda3c..bf004a8 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,10 @@ the network, runs as `nobody`. | `apt_auto_reboot_time_seconds{time,timezone}` | `Automatic-Reboot-Time`, in seconds after midnight in the host time zone (absent when it is `now`) | | `apt_auto_reboot_timezone_offset_seconds{timezone}` | UTC offset of the host time zone | | `node_reboot_scheduled_timestamp_seconds` | time of the reboot planned by unattended-upgrades, 0 if none | +| `apt_upgrade_timer_next_timestamp_seconds` | next run of `apt-daily-upgrade.timer` (absent when systemd can't be asked) | +| `apt_upgrade_timer_last_timestamp_seconds` | last time that timer fired | +| `apt_unattended_upgrades_last_run_timestamp_seconds` | last run of unattended-upgrades that went through (`unattended-upgrades-stamp`); older than the timer's last fire = that run exited early (apt locked…) and planned no reboot | +| `apt_lists_update_timestamp_seconds` | last refresh of the package lists by apt-daily (`update-stamp`) | ## Usage diff --git a/apt-metrics.sh b/apt-metrics.sh index 2a623df..2eb1341 100755 --- a/apt-metrics.sh +++ b/apt-metrics.sh @@ -14,6 +14,25 @@ is_true() { case "$1" in true|yes|on|1) return 0 ;; *) return 1 ;; esac } +# mtime of a file of the host, in seconds since epoch (empty if it is missing) +mtime() { + stat -c %Y "/host$1" 2>/dev/null +} + +# a timestamp property of a host timer, in seconds since epoch (empty if unknown or never) +timer_get() { + v="$(chroot /host systemctl show "$1" -p "$2" --value --timestamp=unix 2>/dev/null)" + case "$v" in @[0-9]*) echo "${v#@}" ;; esac +} + +# metric : nothing when the value is empty +metric() { + [ -n "$3" ] || return 0 + echo "# HELP $1 $2" + echo "# TYPE $1 gauge" + echo "$1 $3" +} + write_metrics() { required=0 [ -f /host/run/reboot-required ] && required=1 @@ -54,7 +73,24 @@ write_metrics() { [ -n "$usec" ] && scheduled=$(( usec / 1000000 )) fi + # the daily run: when the timer fires (6:00 + random delay on Debian), and when + # unattended-upgrade really went through (its stamp: a run that exits early, e.g. because apt + # is locked, fires the timer but leaves the stamp, and does not plan the reboot either) + timer_next="$(timer_get apt-daily-upgrade.timer NextElapseUSecRealtime)" + timer_last="$(timer_get apt-daily-upgrade.timer LastTriggerUSec)" + [ -n "$timer_last" ] || timer_last="$(mtime /var/lib/systemd/timers/stamp-apt-daily-upgrade.timer)" + uu_last="$(mtime /var/lib/apt/periodic/unattended-upgrades-stamp)" + lists_last="$(mtime /var/lib/apt/periodic/update-stamp)" + { + metric apt_upgrade_timer_next_timestamp_seconds \ + 'Next run of apt-daily-upgrade.timer (unattended-upgrades).' "$timer_next" + metric apt_upgrade_timer_last_timestamp_seconds \ + 'Last time apt-daily-upgrade.timer fired.' "$timer_last" + metric apt_unattended_upgrades_last_run_timestamp_seconds \ + 'Last run of unattended-upgrades that went through (its stamp file).' "$uu_last" + metric apt_lists_update_timestamp_seconds \ + 'Last refresh of the package lists by apt-daily (update-stamp).' "$lists_last" echo '# HELP node_reboot_required 1 if a reboot is pending.' echo '# TYPE node_reboot_required gauge' echo "node_reboot_required $required" diff --git a/tests/smoke.sh b/tests/smoke.sh index 2a3a91a..d6486c3 100755 --- a/tests/smoke.sh +++ b/tests/smoke.sh @@ -96,6 +96,11 @@ if wait_metrics /metrics "$m" '^node_reboot_required '; then else echo " skip apt_upgrades_pending: no apt on this host" fi + if systemctl cat apt-daily-upgrade.timer >/dev/null 2>&1; then + expect "$m" '^apt_upgrade_timer_next_timestamp_seconds [0-9.e+]+$' "apt_upgrade_timer_next_timestamp_seconds from the host's systemd" + else + echo " skip apt_upgrade_timer_next_timestamp_seconds: no apt-daily-upgrade.timer on this host" + fi expect "$m" '^node_textfile_scrape_error 0$' "node_textfile_scrape_error 0" else fail "metrics not served on $ADDR" @@ -128,7 +133,11 @@ RUN printf 'APT::Periodic::Update-Package-Lists "1";\nAPT::Periodic::Unattended- > /etc/apt/apt.conf.d/52reboot \ && ln -sf /usr/share/zoneinfo/Europe/Paris /etc/localtime && echo Europe/Paris > /etc/timezone \ && mkdir -p /run/systemd/shutdown && touch /run/reboot-required \ - && printf 'USEC=1893461400000000\nWARN_WALL=1\nMODE=reboot\n' > /run/systemd/shutdown/scheduled + && printf 'USEC=1893461400000000\nWARN_WALL=1\nMODE=reboot\n' > /run/systemd/shutdown/scheduled \ + && mkdir -p /var/lib/apt/periodic /var/lib/systemd/timers \ + && touch -d @1893450000 /var/lib/systemd/timers/stamp-apt-daily-upgrade.timer \ + && touch -d @1893400000 /var/lib/apt/periodic/unattended-upgrades-stamp \ + && touch -d @1893300000 /var/lib/apt/periodic/update-stamp EOF mkdir "$TMP/root" docker create --name "$NAME-host" "$HOST_IMAGE" >/dev/null @@ -148,6 +157,15 @@ if wait_metrics /smoke-metrics "$m" '^apt_upgrades_pending '; then expect "$m" '^apt_auto_reboot_timezone_offset_seconds\{timezone="Europe/Paris"\} (3600|7200)$' "Europe/Paris offset" # node_exporter re-exposes textfile values as floats: 1893461400 -> 1.8934614e+09 expect "$m" '^node_reboot_scheduled_timestamp_seconds (1893461400|1\.8934614e\+09)$' "node_reboot_scheduled_timestamp_seconds" + # no systemd running in this root: the timer is read from its stamp file, the next run is unknown + expect "$m" '^apt_upgrade_timer_last_timestamp_seconds (1893450000|1\.89345e\+09)$' "apt_upgrade_timer_last_timestamp_seconds from the stamp" + expect "$m" '^apt_unattended_upgrades_last_run_timestamp_seconds (1893400000|1\.8934e\+09)$' "apt_unattended_upgrades_last_run_timestamp_seconds" + expect "$m" '^apt_lists_update_timestamp_seconds (1893300000|1\.8933e\+09)$' "apt_lists_update_timestamp_seconds" + if grep -q '^apt_upgrade_timer_next_timestamp_seconds ' "$m"; then + fail "apt_upgrade_timer_next_timestamp_seconds without systemd" + else + pass "no apt_upgrade_timer_next_timestamp_seconds without systemd" + fi else fail "apt metrics not served on $ADDR/smoke-metrics" fi From cbef57a766ea19d6635c4086ce028ecf2281d006 Mon Sep 17 00:00:00 2001 From: parasol-ka Date: Fri, 23 Oct 2026 15:13:30 +0200 Subject: [PATCH 2/2] fix: update timer test on deactivated --- tests/smoke.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/smoke.sh b/tests/smoke.sh index d6486c3..66b1bcd 100755 --- a/tests/smoke.sh +++ b/tests/smoke.sh @@ -96,10 +96,13 @@ if wait_metrics /metrics "$m" '^node_reboot_required '; then else echo " skip apt_upgrades_pending: no apt on this host" fi - if systemctl cat apt-daily-upgrade.timer >/dev/null 2>&1; then + # a stopped timer has no next run (GitHub runners stop and disable apt-daily-upgrade.timer) + if systemctl is-active --quiet apt-daily-upgrade.timer 2>/dev/null; then expect "$m" '^apt_upgrade_timer_next_timestamp_seconds [0-9.e+]+$' "apt_upgrade_timer_next_timestamp_seconds from the host's systemd" + elif grep -q '^apt_upgrade_timer_next_timestamp_seconds ' "$m"; then + fail "apt_upgrade_timer_next_timestamp_seconds with apt-daily-upgrade.timer not active" else - echo " skip apt_upgrade_timer_next_timestamp_seconds: no apt-daily-upgrade.timer on this host" + pass "no apt_upgrade_timer_next_timestamp_seconds: apt-daily-upgrade.timer not active on this host" fi expect "$m" '^node_textfile_scrape_error 0$' "node_textfile_scrape_error 0" else