Skip to content

feat: add more detailed update metrics #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
parasol-ka merged 2 commits into from
Oct 23, 2026
Merged

feat: add more detailed update metrics #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
edb504b
feat: add more detailed update metrics
parasol-ka Oct 23, 2026
cbef57a
fix: update timer test on deactivated
parasol-ka Oct 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ the network, runs as `nobody`.
| `apt_auto_reboot_time_seconds{time,timezone}` | `Automatic-Reboot-Time`, in seconds after midnight in the host time zone (absent when it is `now`) |
| `apt_auto_reboot_timezone_offset_seconds{timezone}` | UTC offset of the host time zone |
| `node_reboot_scheduled_timestamp_seconds` | time of the reboot planned by unattended-upgrades, 0 if none |
| `apt_upgrade_timer_next_timestamp_seconds` | next run of `apt-daily-upgrade.timer` (absent when systemd can't be asked) |
| `apt_upgrade_timer_last_timestamp_seconds` | last time that timer fired |
| `apt_unattended_upgrades_last_run_timestamp_seconds` | last run of unattended-upgrades that went through (`unattended-upgrades-stamp`); older than the timer's last fire = that run exited early (apt locked…) and planned no reboot |
| `apt_lists_update_timestamp_seconds` | last refresh of the package lists by apt-daily (`update-stamp`) |

## Usage

Expand Down
36 changes: 36 additions & 0 deletions apt-metrics.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,25 @@ is_true() {
case "$1" in true|yes|on|1) return 0 ;; *) return 1 ;; esac
}

# mtime of a file of the host, in seconds since epoch (empty if it is missing)
mtime() {
stat -c %Y "/host$1" 2>/dev/null
}

# a timestamp property of a host timer, in seconds since epoch (empty if unknown or never)
timer_get() {
v="$(chroot /host systemctl show "$1" -p "$2" --value --timestamp=unix 2>/dev/null)"
case "$v" in @[0-9]*) echo "${v#@}" ;; esac
}

# metric <name> <help> <value>: nothing when the value is empty
metric() {
[ -n "$3" ] || return 0
echo "# HELP $1 $2"
echo "# TYPE $1 gauge"
echo "$1 $3"
}

write_metrics() {
required=0
[ -f /host/run/reboot-required ] && required=1
Expand Down Expand Up @@ -54,7 +73,24 @@ write_metrics() {
[ -n "$usec" ] && scheduled=$(( usec / 1000000 ))
fi

# the daily run: when the timer fires (6:00 + random delay on Debian), and when
# unattended-upgrade really went through (its stamp: a run that exits early, e.g. because apt
# is locked, fires the timer but leaves the stamp, and does not plan the reboot either)
timer_next="$(timer_get apt-daily-upgrade.timer NextElapseUSecRealtime)"
timer_last="$(timer_get apt-daily-upgrade.timer LastTriggerUSec)"
[ -n "$timer_last" ] || timer_last="$(mtime /var/lib/systemd/timers/stamp-apt-daily-upgrade.timer)"
uu_last="$(mtime /var/lib/apt/periodic/unattended-upgrades-stamp)"
lists_last="$(mtime /var/lib/apt/periodic/update-stamp)"

{
metric apt_upgrade_timer_next_timestamp_seconds \
'Next run of apt-daily-upgrade.timer (unattended-upgrades).' "$timer_next"
metric apt_upgrade_timer_last_timestamp_seconds \
'Last time apt-daily-upgrade.timer fired.' "$timer_last"
metric apt_unattended_upgrades_last_run_timestamp_seconds \
'Last run of unattended-upgrades that went through (its stamp file).' "$uu_last"
metric apt_lists_update_timestamp_seconds \
'Last refresh of the package lists by apt-daily (update-stamp).' "$lists_last"
echo '# HELP node_reboot_required 1 if a reboot is pending.'
echo '# TYPE node_reboot_required gauge'
echo "node_reboot_required $required"
Expand Down
23 changes: 22 additions & 1 deletion tests/smoke.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,14 @@ if wait_metrics /metrics "$m" '^node_reboot_required '; then
else
echo " skip apt_upgrades_pending: no apt on this host"
fi
# a stopped timer has no next run (GitHub runners stop and disable apt-daily-upgrade.timer)
if systemctl is-active --quiet apt-daily-upgrade.timer 2>/dev/null; then
expect "$m" '^apt_upgrade_timer_next_timestamp_seconds [0-9.e+]+$' "apt_upgrade_timer_next_timestamp_seconds from the host's systemd"
elif grep -q '^apt_upgrade_timer_next_timestamp_seconds ' "$m"; then
fail "apt_upgrade_timer_next_timestamp_seconds with apt-daily-upgrade.timer not active"
else
pass "no apt_upgrade_timer_next_timestamp_seconds: apt-daily-upgrade.timer not active on this host"
fi
expect "$m" '^node_textfile_scrape_error 0$' "node_textfile_scrape_error 0"
else
fail "metrics not served on $ADDR"
Expand Down Expand Up @@ -128,7 +136,11 @@ RUN printf 'APT::Periodic::Update-Package-Lists "1";\nAPT::Periodic::Unattended-
> /etc/apt/apt.conf.d/52reboot \
&& ln -sf /usr/share/zoneinfo/Europe/Paris /etc/localtime && echo Europe/Paris > /etc/timezone \
&& mkdir -p /run/systemd/shutdown && touch /run/reboot-required \
&& printf 'USEC=1893461400000000\nWARN_WALL=1\nMODE=reboot\n' > /run/systemd/shutdown/scheduled
&& printf 'USEC=1893461400000000\nWARN_WALL=1\nMODE=reboot\n' > /run/systemd/shutdown/scheduled \
&& mkdir -p /var/lib/apt/periodic /var/lib/systemd/timers \
&& touch -d @1893450000 /var/lib/systemd/timers/stamp-apt-daily-upgrade.timer \
&& touch -d @1893400000 /var/lib/apt/periodic/unattended-upgrades-stamp \
&& touch -d @1893300000 /var/lib/apt/periodic/update-stamp
EOF
mkdir "$TMP/root"
docker create --name "$NAME-host" "$HOST_IMAGE" >/dev/null
Expand All @@ -148,6 +160,15 @@ if wait_metrics /smoke-metrics "$m" '^apt_upgrades_pending '; then
expect "$m" '^apt_auto_reboot_timezone_offset_seconds\{timezone="Europe/Paris"\} (3600|7200)$' "Europe/Paris offset"
# node_exporter re-exposes textfile values as floats: 1893461400 -> 1.8934614e+09
expect "$m" '^node_reboot_scheduled_timestamp_seconds (1893461400|1\.8934614e\+09)$' "node_reboot_scheduled_timestamp_seconds"
# no systemd running in this root: the timer is read from its stamp file, the next run is unknown
expect "$m" '^apt_upgrade_timer_last_timestamp_seconds (1893450000|1\.89345e\+09)$' "apt_upgrade_timer_last_timestamp_seconds from the stamp"
expect "$m" '^apt_unattended_upgrades_last_run_timestamp_seconds (1893400000|1\.8934e\+09)$' "apt_unattended_upgrades_last_run_timestamp_seconds"
expect "$m" '^apt_lists_update_timestamp_seconds (1893300000|1\.8933e\+09)$' "apt_lists_update_timestamp_seconds"
if grep -q '^apt_upgrade_timer_next_timestamp_seconds ' "$m"; then
fail "apt_upgrade_timer_next_timestamp_seconds without systemd"
else
pass "no apt_upgrade_timer_next_timestamp_seconds without systemd"
fi
else
fail "apt metrics not served on $ADDR/smoke-metrics"
fi
Expand Down
Loading