diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e81d7ee..eca126d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,8 +13,15 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - run: shellcheck -S warning eb-keep-logs tests/test_eb_keep_logs.sh tests/corpus/run.sh tests/corpus/make-fixtures.sh + - name: Install promtool 3.7.3 + run: | + curl -fsSLO https://github.com/prometheus/prometheus/releases/download/v3.7.3/prometheus-3.7.3.linux-amd64.tar.gz + echo 'fc9e5da126817438cf2820d9a7206c75e3122802ba8d20add3a6219a59ca6913 prometheus-3.7.3.linux-amd64.tar.gz' | sha256sum -c - + tar -xzf prometheus-3.7.3.linux-amd64.tar.gz + sudo install -m 0755 prometheus-3.7.3.linux-amd64/promtool /usr/local/bin/promtool + - run: shellcheck -S warning eb-keep-logs tests/test_eb_keep_logs.sh tests/test_metrics.sh tests/corpus/run.sh tests/corpus/make-fixtures.sh - run: bash tests/test_eb_keep_logs.sh + - run: bash tests/test_metrics.sh - name: no site names in the tree run: | ! git grep -nIiE 'snellius|snel-|surf\.nl|/gpfs/|/projects/0|rg\.terra|scratch-shared|scratch-node' -- . ':!.github/workflows/ci.yml' @@ -34,8 +41,20 @@ jobs: - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 with: version: 0.17.1 + - name: Install promtool 3.7.3 + run: | + case '${{ runner.arch }}' in + X64) arch=amd64; sha=fc9e5da126817438cf2820d9a7206c75e3122802ba8d20add3a6219a59ca6913 ;; + ARM64) arch=arm64; sha=6af448fa4f3640eb9bd51ce71575689a1ef79843b1cf08ef1fcc7ff9a1928bfd ;; + *) echo 'unsupported runner architecture: ${{ runner.arch }}' >&2; exit 1 ;; + esac + archive="prometheus-3.7.3.linux-${arch}.tar.gz" + curl -fsSLO "https://github.com/prometheus/prometheus/releases/download/v3.7.3/${archive}" + echo "${sha} ${archive}" | sha256sum -c - + tar -xzf "${archive}" + sudo install -m 0755 "prometheus-3.7.3.linux-${arch}/promtool" /usr/local/bin/promtool - run: zig fmt --check build.zig build.zig.zon src - - run: shellcheck -S warning tests/differential.sh tests/test_why.sh tests/test_json_schemas.sh + - run: shellcheck -S warning tests/differential.sh tests/test_metrics.sh tests/test_why.sh tests/test_json_schemas.sh - run: pip install jsonschema - run: zig build test --summary all - run: zig build -Drelease=true -Dcommit=${{ github.sha }} && zig-out/bin/eb-keep-logs --version diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5e2c18b..5e9ea82 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,6 +25,18 @@ jobs: - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 with: version: 0.17.1 + - name: Install promtool 3.7.3 + run: | + case '${{ matrix.arch }}' in + x86_64) prom_arch=amd64; sha=fc9e5da126817438cf2820d9a7206c75e3122802ba8d20add3a6219a59ca6913 ;; + aarch64) prom_arch=arm64; sha=6af448fa4f3640eb9bd51ce71575689a1ef79843b1cf08ef1fcc7ff9a1928bfd ;; + esac + archive="prometheus-3.7.3.linux-${prom_arch}.tar.gz" + curl -fsSLO "https://github.com/prometheus/prometheus/releases/download/v3.7.3/${archive}" + echo "${sha} ${archive}" | sha256sum -c - + tar -xzf "${archive}" + sudo install -m 0755 "prometheus-3.7.3.linux-${prom_arch}/promtool" /usr/local/bin/promtool + - run: pip install jsonschema - run: zig build test --summary all - name: build run: | diff --git a/build.zig b/build.zig index 44e9ce1..b74dc73 100644 --- a/build.zig +++ b/build.zig @@ -112,6 +112,11 @@ pub fn build(b: *std.Build) void { stable_output_test.setCwd(b.path(".")); stable_output_test.expectExitCode(0); stable_output_test.has_side_effects = true; + const metrics_test = b.addSystemCommand(&.{ "bash", "-c", "TOOL_BIN=\"$0\" exec bash tests/test_metrics.sh" }); + metrics_test.addArtifactArg(exe); + metrics_test.setCwd(b.path(".")); + metrics_test.expectExitCode(0); + metrics_test.has_side_effects = true; const test_step = b.step("test", "Run the unit tests, the command-line suites and the differential run"); test_step.dependOn(&b.addRunArtifact(unit).step); @@ -128,4 +133,5 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&test_bundle_test.step); test_step.dependOn(&lifecycle_test.step); test_step.dependOn(&stable_output_test.step); + test_step.dependOn(&metrics_test.step); } diff --git a/changelog.d/+prometheus-metrics.feature.md b/changelog.d/+prometheus-metrics.feature.md new file mode 100644 index 0000000..03a4a3e --- /dev/null +++ b/changelog.d/+prometheus-metrics.feature.md @@ -0,0 +1 @@ +Add opt-in atomic Prometheus textfile metrics for kept logs, kept bytes, and the last completed rotation. diff --git a/docs/orgmode/explanation/local-time.org b/docs/orgmode/explanation/local-time.org index 0e8d244..1e27d48 100644 --- a/docs/orgmode/explanation/local-time.org +++ b/docs/orgmode/explanation/local-time.org @@ -28,7 +28,8 @@ The Zig build has to give the same ages without =date=: it is one static binary 4. Where a change to daylight saving time skips midnight, the clock jumping from 00:00 to 01:00, that date's midnight does not exist. =date -d= refuses such a date, and =rotate= skips the run, as the shell script did. -=EB_KEEP_LOGS_NOW= replaces now with a time of your choosing, in seconds since the epoch; nothing else changes. +=EB_KEEP_LOGS_NOW= replaces now for these age calculations with a time of your choosing, in seconds since the epoch. +It does not replace the actual completion time written to a metrics file. ** What was considered and turned down diff --git a/docs/orgmode/howto/export-prometheus-metrics.org b/docs/orgmode/howto/export-prometheus-metrics.org new file mode 100644 index 0000000..0d8fb47 --- /dev/null +++ b/docs/orgmode/howto/export-prometheus-metrics.org @@ -0,0 +1,79 @@ +#+TITLE: Export metrics to Prometheus +#+OPTIONS: toc:nil todo:nil num:nil author:nil title:nil +#+STARTUP: content + +* Export metrics to Prometheus + +Use the node_exporter textfile collector when =eb-keep-logs= and node_exporter run on the same machine. +The metrics output is opt-in, so existing commands write no metrics file. + +** Choose the collector directory + +Start node_exporter with a textfile collector directory: + +#+begin_src sh +node_exporter --collector.textfile.directory=/var/lib/node_exporter/textfile_collector +#+end_src + +For a systemd service, add the option to the service's command line and restart the service. +As =root=, create the collector directory with the build account as its owner and node_exporter's group as its group: + +#+begin_src sh +install -d -o build-account -g node-exporter -m 0750 \ + /var/lib/node_exporter/textfile_collector +#+end_src + +Replace =build-account= with the account that keeps and rotates the logs, and =node-exporter= with the group of the node_exporter service account. +The owner can create the sidecar lock and temporary file, rename the temporary file atomically, and remove stale temporary files. +The service group can traverse and read the directory. +Do not create an empty =.prom= file: the first command that uses =--metrics-file= creates a complete exposition with mode 0644. + +** Record kept logs + +Pass the same metrics path to each =failed= invocation: + +#+begin_src sh +eb-keep-logs failed \ + --metrics-file /var/lib/node_exporter/textfile_collector/eb-keep-logs.prom \ + "${status}" "${easyconfig_name}" "${log_dir}" "${tmp_log_dir}" +#+end_src + +Each successfully copied log increments =eb_keep_logs_logs_kept_total=. +Its written size increments =eb_keep_logs_bytes_kept_total=. +A retry that copies a log again is another keep operation and increments both counters again. + +** Record the last rotation + +Pass the path to =rotate= too: + +#+begin_src sh +eb-keep-logs rotate /srv/build-logs \ + --compress-after 30 --delete-after 90 \ + --metrics-file /var/lib/node_exporter/textfile_collector/eb-keep-logs.prom +#+end_src + +After rotation completes, =eb_keep_logs_last_prune_timestamp_seconds= is the actual completion Unix timestamp, even when =EB_KEEP_LOGS_NOW= set a different time for run-age calculations. +The time is chosen while holding the metrics lock, and an older completion never replaces a newer one when rotations finish concurrently. +The counters remain unchanged. +A rotation that gives up waiting for its lock does not update the timestamp. + +The tool locks a sidecar file, writes a temporary file in the collector directory, and renames it over the metrics file. +The collector therefore reads either the previous complete exposition or the new one. + +** Check the result + +Validate the Prometheus text exposition: + +#+begin_src sh +promtool check metrics < /var/lib/node_exporter/textfile_collector/eb-keep-logs.prom +#+end_src + +Then inspect node_exporter's metrics endpoint: + +#+begin_src sh +curl -fsS http://127.0.0.1:9100/metrics | + grep '^eb_keep_logs_' +#+end_src + +All three names start with =eb_keep_logs_=. +When =--metrics-file= is omitted, none of them is written. diff --git a/docs/orgmode/howto/index.org b/docs/orgmode/howto/index.org index 06861a0..e575814 100644 --- a/docs/orgmode/howto/index.org +++ b/docs/orgmode/howto/index.org @@ -24,4 +24,5 @@ The title names the task. rotate-a-log-tree mark-licensed-logs emit-syslog-events + export-prometheus-metrics #+end_export diff --git a/docs/orgmode/reference/cli.org b/docs/orgmode/reference/cli.org index 4b3f15c..2f70359 100644 --- a/docs/orgmode/reference/cli.org +++ b/docs/orgmode/reference/cli.org @@ -10,8 +10,8 @@ This page describes the Zig build; the shell script =eb-keep-logs= in the reposi ** Synopsis #+begin_example -eb-keep-logs failed [--syslog TARGET] STATUS NAME LOGDIR [TMP_LOGDIR] -eb-keep-logs rotate ROOT --compress-after DAYS [--delete-after DAYS] [--lock-timeout SECONDS] [--syslog TARGET] +eb-keep-logs failed [--syslog TARGET] [--metrics-file PATH] STATUS NAME LOGDIR [TMP_LOGDIR] +eb-keep-logs rotate ROOT --compress-after DAYS [--delete-after DAYS] [--lock-timeout SECONDS] [--syslog TARGET] [--metrics-file PATH] eb-keep-logs why [RUN | --latest] [--target TARGET] [--root ROOT] [--json] eb-keep-logs history PACKAGE [--target TARGET] [--root ROOT] [--json] eb-keep-logs loads --module NAME [--json] FILE @@ -45,6 +45,7 @@ eb-keep-logs --version | =--delete-after DAYS=, =--delete-after=DAYS= | =rotate= | Remove whole runs at least =DAYS= old. =DAYS= must be larger than =--compress-after=. Without it, or with an empty value, no run is removed. | | =--lock-timeout SECONDS=, =--lock-timeout=SECONDS= | =rotate= | How long to wait for the lock another =rotate= of =ROOT= holds, in seconds, decimal digits; =0= does not wait. Without it, or with an empty value, 600. A value larger than 999999999, which is over 31 years, waits 999999999 seconds. | | =--syslog TARGET=, =--syslog=TARGET= | =failed=, =rotate= | Emit an RFC 5424 or native =systemd-journald= event to =TARGET= (=journald=, =syslog=, =unix:///path=, =journal+unix:///path=, =udp://host:port=, or =tcp://host:port=); overrides =EB_KEEP_LOGS_SYSLOG=. An empty value disables emission. | +| =--metrics-file PATH=, =--metrics-file=PATH= | =failed=, =rotate= | Atomically update the Prometheus text exposition at =PATH=. Without it, no metrics file is read or written. | | =--latest= | =why= | Read the newest run under =--root=, by its name; with =--target=, the newest that has that target. The default without =RUN=. | | =--target TARGET=, =--target=TARGET= | =why=, =history= | Read only the builds in the run's directory =TARGET=; for =history=, in each run's, a run without one passed over. Without it, those in the run's own directory and then in each directory in it, by name. | | =--root ROOT=, =--root=ROOT= | =why=, =history= | The directory that holds the runs, directories named =YYYYMMDD_HHMMSS=, not symbolic links. Without it, the current directory. | @@ -77,7 +78,7 @@ eb-keep-logs --version An option given twice takes its last value. - =why='s options and =RUN=, and =history='s and =PACKAGE=, come in any order, and an option given twice takes its last value; the value of an option is the next argument, whatever it is, and an empty one is refused. - =-h=, =--help= and =--version= count only in the place of the command. -- =failed= takes no options. +- =failed='s options and positional arguments may appear in any order. ** Environment @@ -213,6 +214,23 @@ What =rotate= cannot do it reports on stderr as =eb-keep-logs: MESSAGE=, and goe | =cannot sync a .licensed-logs: ERROR= | A list was replaced, but its directory could not be synced; after a crash the old list may be back. | | =cannot delete ROOT/RUN: ERROR= | The run could not be removed completely; what was removed stays removed. | +*** Metrics + +With =--metrics-file PATH=, =failed= and =rotate= update one Prometheus text exposition file. +The file is written to a temporary file in =PATH='s directory, synced, and renamed over =PATH= while a sidecar lock is held. +An existing valid file supplies the counters and timestamp that the other command owns. +An invalid file is left unchanged and reported on stderr. + +| Metric | Type | Value | +|--------+------+-------| +| =eb_keep_logs_logs_kept_total= | counter | Successful log-copy operations by =failed=, including a retry that copies the same destination again. | +| =eb_keep_logs_bytes_kept_total= | counter | Bytes in those successfully written copies. | +| =eb_keep_logs_last_prune_timestamp_seconds= | gauge | Actual Unix timestamp when =rotate= most recently completed, including a rotation that found nothing to change. It is chosen under the metrics lock, never moves backwards when rotations complete out of order, is independent of =EB_KEEP_LOGS_NOW=, and is zero until one completes. | + +Metrics are values this tool observes while keeping and rotating logs. +They do not describe dependency gaps, rebuild closures, failure classes, or other stack analysis. +See [[file:../howto/export-prometheus-metrics.org][Export metrics to Prometheus]] for the node_exporter textfile collector setup. + *** why =why= only reads. @@ -507,7 +525,7 @@ The usage =-h=, =--help= and no arguments print on stdout, the same from the she eb-keep-logs: keep the logs of EasyBuild builds that a build pipeline would otherwise lose, and keep the pipeline's log tree bounded. - eb-keep-logs failed [--syslog TARGET] STATUS NAME LOGDIR [TMP_LOGDIR] + eb-keep-logs failed [--syslog TARGET] [--metrics-file PATH] STATUS NAME LOGDIR [TMP_LOGDIR] After `eb` returned STATUS for easyconfig NAME: when STATUS is not 0, copy the EasyBuild session log `eb --last-log` names, the failing command's output and a meson testlog next to LOGDIR/NAME.stdout.log, @@ -518,8 +536,12 @@ otherwise lose, and keep the pipeline's log tree bounded. --syslog TARGET, --syslog=TARGET Emit an RFC 5424 or native systemd-journald event when logs are kept or none are found. Default: EB_KEEP_LOGS_SYSLOG, else none. + --metrics-file PATH, --metrics-file=PATH + Atomically update Prometheus counters for kept logs and bytes. + Default: none. eb-keep-logs rotate ROOT --compress-after DAYS [--delete-after DAYS] [--lock-timeout SECONDS] [--syslog TARGET] + [--metrics-file PATH] In the run directories directly under ROOT (named YYYYMMDD_HHMMSS), gzip the *.log and *.txt files of old runs and remove older runs whole, leaving alone a run that is still written into. A run's age is @@ -541,6 +563,8 @@ otherwise lose, and keep the pipeline's log tree bounded. Emit an RFC 5424 or native systemd-journald event with the compressed and deleted counts. Default: EB_KEEP_LOGS_SYSLOG, else none. + --metrics-file PATH, --metrics-file=PATH + Atomically record when this rotation completed. Default: none. eb-keep-logs why [RUN | --latest] [--target TARGET] [--root ROOT] [--json] Say why the builds of a run failed: for each failed easyconfig, its package and easyconfig, the EasyBuild step it failed in, EasyBuild's diff --git a/eb-keep-logs b/eb-keep-logs index d53c77f..5dd6d89 100755 --- a/eb-keep-logs +++ b/eb-keep-logs @@ -17,7 +17,7 @@ usage() { eb-keep-logs: keep the logs of EasyBuild builds that a build pipeline would otherwise lose, and keep the pipeline's log tree bounded. - eb-keep-logs failed [--syslog TARGET] STATUS NAME LOGDIR [TMP_LOGDIR] + eb-keep-logs failed [--syslog TARGET] [--metrics-file PATH] STATUS NAME LOGDIR [TMP_LOGDIR] After `eb` returned STATUS for easyconfig NAME: when STATUS is not 0, copy the EasyBuild session log `eb --last-log` names, the failing command's output and a meson testlog next to LOGDIR/NAME.stdout.log, @@ -28,8 +28,12 @@ otherwise lose, and keep the pipeline's log tree bounded. --syslog TARGET, --syslog=TARGET Emit an RFC 5424 or native systemd-journald event when logs are kept or none are found. Default: EB_KEEP_LOGS_SYSLOG, else none. + --metrics-file PATH, --metrics-file=PATH + Atomically update Prometheus counters for kept logs and bytes. + Default: none. eb-keep-logs rotate ROOT --compress-after DAYS [--delete-after DAYS] [--lock-timeout SECONDS] [--syslog TARGET] + [--metrics-file PATH] In the run directories directly under ROOT (named YYYYMMDD_HHMMSS), gzip the *.log and *.txt files of old runs and remove older runs whole, leaving alone a run that is still written into. A run's age is @@ -51,6 +55,8 @@ otherwise lose, and keep the pipeline's log tree bounded. Emit an RFC 5424 or native systemd-journald event with the compressed and deleted counts. Default: EB_KEEP_LOGS_SYSLOG, else none. + --metrics-file PATH, --metrics-file=PATH + Atomically record when this rotation completed. Default: none. eb-keep-logs why [RUN | --latest] [--target TARGET] [--root ROOT] [--json] Say why the builds of a run failed: for each failed easyconfig, its package and easyconfig, the EasyBuild step it failed in, EasyBuild's @@ -535,8 +541,140 @@ _emit_failed_syslog() { "J:EB_KEEP_REASON=${reason}" "${extra[@]}" } +# Add two unsigned decimal integers without using Bash's signed arithmetic. +# Refuse a result beyond the u64 range used by the Zig implementation. +_add_u64() { + local a="$1" b="$2" out="" carry=0 da db sum + local i j max=18446744073709551615 + a="${a#"${a%%[!0]*}"}"; a="${a:-0}" + b="${b#"${b%%[!0]*}"}"; b="${b:-0}" + i=${#a}; j=${#b} + while [ "${i}" -gt 0 ] || [ "${j}" -gt 0 ] || [ "${carry}" -gt 0 ]; do + da=0 + db=0 + if [ "${i}" -gt 0 ]; then i=$((i - 1)); da="${a:i:1}"; fi + if [ "${j}" -gt 0 ]; then j=$((j - 1)); db="${b:j:1}"; fi + sum=$((da + db + carry)) + out="$((sum % 10))${out}" + carry=$((sum / 10)) + done + # Decimal strings of equal length are ordered lexically. + # shellcheck disable=SC2071 + if [ "${#out}" -gt "${#max}" ] || + { [ "${#out}" -eq "${#max}" ] && [[ "${out}" > "${max}" ]]; }; then + return 1 + fi + printf '%s\n' "${out}" +} + +_u64_gt() { + local a="$1" b="$2" + a="${a#"${a%%[!0]*}"}"; a="${a:-0}" + b="${b#"${b%%[!0]*}"}"; b="${b:-0}" + [ "${#a}" -gt "${#b}" ] && return 0 + [ "${#a}" -lt "${#b}" ] && return 1 + # Decimal strings of equal length are ordered lexically. + # shellcheck disable=SC2071 + [[ "${a}" > "${b}" ]] +} + +_update_metrics() { + local file="$1" add_logs="$2" add_bytes="$3" prune="$4" + [ -n "${file}" ] || return 0 + local dir base lock fd tmp logs=0 bytes=0 last=0 value new_logs new_bytes + dir="${file%/*}"; [ "${dir}" != "${file}" ] || dir=. + base="${file##*/}" + if [ -z "${base}" ] || [ ! -d "${dir}" ]; then + echo "eb-keep-logs: cannot update metrics file ${file}: invalid path" >&2 + return 0 + fi + lock="${dir}/.${base}.lock" + if [ -L "${lock}" ] || { [ -e "${lock}" ] && [ ! -f "${lock}" ]; } || + ! { exec {fd}>>"${lock}"; } 2>/dev/null || ! flock "${fd}"; then + echo "eb-keep-logs: cannot update metrics file ${file}: cannot lock it" >&2 + return 0 + fi + if [ -e "${file}" ]; then + if [ ! -f "${file}" ] || [ -L "${file}" ]; then + echo "eb-keep-logs: cannot update metrics file ${file}: not a regular file" >&2 + exec {fd}>&- + return 0 + fi + value=$(awk '$1 == "eb_keep_logs_logs_kept_total" { if (++n > 1 || $2 !~ /^[0-9]+$/) exit 2; v=$2 } END { if (n) print v }' "${file}") + if [ "$?" -ne 0 ]; then + echo "eb-keep-logs: cannot update metrics file ${file}: invalid metrics" >&2 + exec {fd}>&- + return 0 + fi + logs=${value:-0} + value=$(awk '$1 == "eb_keep_logs_bytes_kept_total" { if (++n > 1 || $2 !~ /^[0-9]+$/) exit 2; v=$2 } END { if (n) print v }' "${file}") + if [ "$?" -ne 0 ]; then + echo "eb-keep-logs: cannot update metrics file ${file}: invalid metrics" >&2 + exec {fd}>&- + return 0 + fi + bytes=${value:-0} + value=$(awk '$1 == "eb_keep_logs_last_prune_timestamp_seconds" { if (++n > 1 || $2 !~ /^[0-9]+$/) exit 2; v=$2 } END { if (n) print v }' "${file}") + if [ "$?" -ne 0 ]; then + echo "eb-keep-logs: cannot update metrics file ${file}: invalid metrics" >&2 + exec {fd}>&- + return 0 + fi + last=${value:-0} + fi + if ! new_logs=$(_add_u64 "${logs}" "${add_logs}") || + ! new_bytes=$(_add_u64 "${bytes}" "${add_bytes}"); then + echo "eb-keep-logs: cannot update metrics file ${file}: counter overflow" >&2 + exec {fd}>&- + return 0 + fi + logs="${new_logs}" + bytes="${new_bytes}" + # Choose completion time only after taking the metrics lock. Keep a later + # completion already recorded by a concurrent, out-of-order rotation. + [ "${prune}" != "now" ] || prune=$(date +%s) + if [ -n "${prune}" ] && _u64_gt "${prune}" "${last}"; then + last="${prune}" + fi + tmp=$(mktemp -- "${dir}/.${base}.XXXXXXXX.tmp") || { + echo "eb-keep-logs: cannot update metrics file ${file}: cannot create temporary file" >&2 + exec {fd}>&- + return 0 + } + if ! cat > "${tmp}" <&- + return 0 + fi + if { [ -e "${file}" ] && ! chmod --reference="${file}" -- "${tmp}"; } || + { [ ! -e "${file}" ] && ! chmod 644 "${tmp}"; }; then + echo "eb-keep-logs: cannot update metrics file ${file}: cannot set temporary file mode" >&2 + rm -f -- "${tmp}" + exec {fd}>&- + return 0 + fi + if ! sync -f "${tmp}" || ! mv -f -- "${tmp}" "${file}"; then + echo "eb-keep-logs: cannot update metrics file ${file}: cannot replace it" >&2 + rm -f -- "${tmp}" + elif ! sync -f "${dir}"; then + echo "eb-keep-logs: cannot update metrics file ${file}: cannot sync its directory" >&2 + fi + exec {fd}>&- +} + cmd_failed() { - local pos=() syslog_opt="" have_syslog=0 + local pos=() syslog_opt="" have_syslog=0 metrics_file="" have_metrics=0 while [ $# -gt 0 ]; do case "$1" in --syslog) @@ -544,6 +682,11 @@ cmd_failed() { syslog_opt="$2"; have_syslog=1; shift 2 ;; --syslog=*) syslog_opt="${1#*=}"; have_syslog=1; shift ;; + --metrics-file) + [ $# -ge 2 ] || die "--metrics-file needs a path" + metrics_file="$2"; have_metrics=1; shift 2 ;; + --metrics-file=*) + metrics_file="${1#*=}"; have_metrics=1; shift ;; *) pos+=("$1"); shift ;; esac done @@ -552,6 +695,7 @@ cmd_failed() { local src dest [[ "${status}" =~ ^[0-9]+$ ]] || die "STATUS must be a number, not '${status}'" [ -d "${logdir}" ] || die "LOGDIR ${logdir} is not a directory" + [ "${have_metrics}" -eq 0 ] || [ -n "${metrics_file}" ] || die "--metrics-file needs a path" local syslog_target="" _resolve_syslog_pen if [ "${have_syslog}" -eq 1 ]; then @@ -563,6 +707,7 @@ cmd_failed() { fi logdir=$(_joinable "${logdir}") if [ "${status}" -eq 0 ]; then + _update_metrics "${metrics_file}" 0 0 "" return 0 fi # The failed lines the kept copies hold are also in the build's own stdout @@ -586,6 +731,7 @@ cmd_failed() { if ! command -v eb >/dev/null 2>&1; then echo "No EasyBuild log to keep for ${name}: eb is not on PATH." _emit_failed_syslog "${syslog_target}" "${status}" "${name}" "${logdir}" 0 0 0 "no-eb" + _update_metrics "${metrics_file}" 0 0 "" return 0 fi local rc=0 @@ -597,17 +743,19 @@ cmd_failed() { if [ "${rc}" -ne 0 ]; then echo "No EasyBuild log to keep for ${name}: eb --last-log exited ${rc}." _emit_failed_syslog "${syslog_target}" "${status}" "${name}" "${logdir}" 0 0 0 "last-log-failed" + _update_metrics "${metrics_file}" 0 0 "" return 0 fi if [ -z "${src}" ] || [ ! -f "${src}" ]; then echo "No EasyBuild log to keep for ${name}." _emit_failed_syslog "${syslog_target}" "${status}" "${name}" "${logdir}" 0 0 0 "no-log" + _update_metrics "${metrics_file}" 0 0 "" return 0 fi # Without TMP_LOGDIR, `eb --last-log` names the newest log in EasyBuild's # default place, which can be a later build's. Nothing is taken from that # log; the build's own stdout log still names its failing command. - local foreign="" ebout_kept=0 cmd_kept=0 meson_kept=0 + local foreign="" ebout_kept=0 cmd_kept=0 meson_kept=0 kept_bytes=0 if [ -z "${explicit_logdir}" ] && grep -q 'processing EasyBuild easyconfig ' "${src}" && ! grep -qF -- "${name}.eb" "${src}"; then echo "No EasyBuild log to keep for ${name}: ${src} is another build's log." @@ -618,6 +766,7 @@ cmd_failed() { _private "${dest}" "${logdir}" echo "Failed build log: ${dest}" ebout_kept=1 + kept_bytes=$(_add_u64 "${kept_bytes}" "$(stat -c %s "${dest}")") fi # The failing command's own output holds the diagnostic beyond EasyBuild's @@ -658,6 +807,7 @@ cmd_failed() { _list "${dest}" "${logdir}" echo "Failed command output: ${dest}" cmd_kept=$((cmd_kept + 1)) + kept_bytes=$(_add_u64 "${kept_bytes}" "$(stat -c %s "${dest}")") done fi @@ -665,6 +815,7 @@ cmd_failed() { local reason="other-build" [ "${cmd_kept}" -gt 0 ] && reason="kept" _emit_failed_syslog "${syslog_target}" "${status}" "${name}" "${logdir}" 0 "${cmd_kept}" 0 "${reason}" + _update_metrics "${metrics_file}" "${cmd_kept}" "${kept_bytes}" "" return 0 fi # A meson test step prints only the last 100 lines of a failing test; the @@ -679,10 +830,12 @@ cmd_failed() { _private "${dest}" "${logdir}" echo "Failed meson test log: ${dest}" meson_kept=1 + kept_bytes=$(_add_u64 "${kept_bytes}" "$(stat -c %s "${dest}")") fi local reason="no-log" [ $((ebout_kept + cmd_kept + meson_kept)) -gt 0 ] && reason="kept" _emit_failed_syslog "${syslog_target}" "${status}" "${name}" "${logdir}" "${ebout_kept}" "${cmd_kept}" "${meson_kept}" "${reason}" + _update_metrics "${metrics_file}" "$((ebout_kept + cmd_kept + meson_kept))" "${kept_bytes}" "" return 0 } @@ -758,7 +911,7 @@ _lock_rotate() { } cmd_rotate() { - local root="" compress_after="" delete_after="" lock_timeout="" syslog_opt="" have_syslog=0 + local root="" compress_after="" delete_after="" lock_timeout="" syslog_opt="" have_syslog=0 metrics_file="" have_metrics=0 while [ $# -gt 0 ]; do case "$1" in --compress-after) compress_after="${2:-}"; shift 2 ;; @@ -773,12 +926,17 @@ cmd_rotate() { [ $# -ge 2 ] || die "--syslog needs a target" syslog_opt="$2"; have_syslog=1; shift 2 ;; --syslog=*) syslog_opt="${1#*=}"; have_syslog=1; shift ;; + --metrics-file) + [ $# -ge 2 ] || die "--metrics-file needs a path" + metrics_file="$2"; have_metrics=1; shift 2 ;; + --metrics-file=*) metrics_file="${1#*=}"; have_metrics=1; shift ;; -*) die "unknown option $1" ;; *) [ -z "${root}" ] || die "rotate takes one ROOT"; root="$1"; shift ;; esac done [ -n "${root}" ] || die "rotate needs ROOT" [ -d "${root}" ] || die "ROOT ${root} is not a directory" + [ "${have_metrics}" -eq 0 ] || [ -n "${metrics_file}" ] || die "--metrics-file needs a path" [[ "${compress_after}" =~ ^[0-9]+$ ]] || die "--compress-after needs a number of days" if [ -n "${delete_after}" ]; then [[ "${delete_after}" =~ ^[0-9]+$ ]] || die "--delete-after needs a number of days" @@ -840,6 +998,7 @@ cmd_rotate() { _emit_syslog "${syslog_target}" 6 "EB_ROTATE_DONE" "${summary}" \ "P:cmd=rotate" "P:root=${root}" "P:compressed=${compressed}" "P:deleted=${deleted}" \ "J:EB_KEEP_CMD=rotate" "J:EB_ROTATE_ROOT=${root}" "J:EB_ROTATE_COMPRESSED=${compressed}" "J:EB_ROTATE_DELETED=${deleted}" + _update_metrics "${metrics_file}" 0 0 "now" } case "${1:-}" in diff --git a/src/main.zig b/src/main.zig index 9b97a71..faa883d 100644 --- a/src/main.zig +++ b/src/main.zig @@ -20,6 +20,7 @@ const usage_cmd = @import("usage.zig"); const test_bundle = @import("test_bundle.zig"); const lifecycle = @import("lifecycle.zig"); const syslog = @import("syslog.zig"); +const metrics = @import("metrics.zig"); const Allocator = std.mem.Allocator; const tool = "eb-keep-logs"; @@ -33,7 +34,7 @@ const usage_text = \\eb-keep-logs: keep the logs of EasyBuild builds that a build pipeline would \\otherwise lose, and keep the pipeline's log tree bounded. \\ - \\ eb-keep-logs failed [--syslog TARGET] STATUS NAME LOGDIR [TMP_LOGDIR] + \\ eb-keep-logs failed [--syslog TARGET] [--metrics-file PATH] STATUS NAME LOGDIR [TMP_LOGDIR] \\ After `eb` returned STATUS for easyconfig NAME: when STATUS is not 0, \\ copy the EasyBuild session log `eb --last-log` names, the failing \\ command's output and a meson testlog next to LOGDIR/NAME.stdout.log, @@ -44,8 +45,12 @@ const usage_text = \\ --syslog TARGET, --syslog=TARGET \\ Emit an RFC 5424 or native systemd-journald event when logs are \\ kept or none are found. Default: EB_KEEP_LOGS_SYSLOG, else none. + \\ --metrics-file PATH, --metrics-file=PATH + \\ Atomically update Prometheus counters for kept logs and bytes. + \\ Default: none. \\ eb-keep-logs rotate ROOT --compress-after DAYS [--delete-after DAYS] \\ [--lock-timeout SECONDS] [--syslog TARGET] + \\ [--metrics-file PATH] \\ In the run directories directly under ROOT (named YYYYMMDD_HHMMSS), \\ gzip the *.log and *.txt files of old runs and remove older runs \\ whole, leaving alone a run that is still written into. A run's age is @@ -67,6 +72,8 @@ const usage_text = \\ Emit an RFC 5424 or native systemd-journald event with the \\ compressed and deleted counts. Default: EB_KEEP_LOGS_SYSLOG, \\ else none. + \\ --metrics-file PATH, --metrics-file=PATH + \\ Atomically record when this rotation completed. Default: none. \\ eb-keep-logs why [RUN | --latest] [--target TARGET] [--root ROOT] [--json] \\ Say why the builds of a run failed: for each failed easyconfig, its \\ package and easyconfig, the EasyBuild step it failed in, EasyBuild's @@ -467,11 +474,12 @@ fn copyDurable(dir: fsx.fd_t, name: []const u8, src: fsx.fd_t, src_stat: fsx.Sta /// Copy an open file into LOGDIR as `name`, set its mode to `closed_mode` /// or, when that is null, take others' bits off the copy's own mode (chmod -/// o-rwx), and list it. False when the copy failed. -fn keep(c: Ctx, logdir: fsx.fd_t, logdir_s: []const u8, src: fsx.fd_t, src_stat: fsx.Stat, name: []const u8, comptime closed_mode: ?u32) bool { +/// o-rwx), and list it. Return the written byte count, or null when the copy +/// failed. +fn keep(c: Ctx, logdir: fsx.fd_t, logdir_s: []const u8, src: fsx.fd_t, src_stat: fsx.Stat, name: []const u8, comptime closed_mode: ?u32) ?u64 { const fd = copyDurable(logdir, name, src, src_stat) catch |err| { warn("cannot keep {s}/{s}: {t}", .{ logdir_s, name, err }); - return false; + return null; }; defer sys.close(fd); // As chmod o-rwx fails and leaves the mode when the file cannot be @@ -482,7 +490,7 @@ fn keep(c: Ctx, logdir: fsx.fd_t, logdir_s: []const u8, src: fsx.fd_t, src_stat: }; if (mode) |m| sys.fchmod(fd, @intCast(m)) catch |err| warn("cannot close {s}/{s} to others: {t}", .{ logdir_s, name, err }); listKept(c, logdir, fd); - return true; + return @intCast((fsx.fstat(fd) catch return 0).size); } fn rewind(fd: fsx.fd_t) void { @@ -658,10 +666,29 @@ fn emitRotateSyslog( } } +fn updateMetrics(c: Ctx, path: ?[]const u8, logs_kept: usize, bytes_kept: u64, record_completion: bool) void { + const p = path orelse return; + metrics.update(c.gpa, p, @intCast(logs_kept), bytes_kept, record_completion) catch |err| + warn("cannot update metrics file {s}: {t}", .{ p, err }); +} + +const KeepStats = struct { + logs: usize = 0, + bytes: u64 = 0, + + fn add(s: *KeepStats, bytes: ?u64) void { + if (bytes) |n| { + s.logs += 1; + s.bytes += n; + } + } +}; + fn cmdFailed(c: Ctx, args: []const [:0]const u8) u8 { var pos: std.ArrayList([]const u8) = .empty; defer pos.deinit(c.gpa); var syslog_v: ?[]const u8 = null; + var metrics_file: ?[]const u8 = null; var i: usize = 0; while (i < args.len) : (i += 1) { const a = args[i]; @@ -671,6 +698,12 @@ fn cmdFailed(c: Ctx, args: []const [:0]const u8) u8 { syslog_v = args[i]; } else if (std.mem.startsWith(u8, a, "--syslog=")) { syslog_v = a["--syslog=".len..]; + } else if (std.mem.eql(u8, a, "--metrics-file")) { + if (i + 1 >= args.len) die("--metrics-file needs a path", .{}); + i += 1; + metrics_file = args[i]; + } else if (std.mem.startsWith(u8, a, "--metrics-file=")) { + metrics_file = a["--metrics-file=".len..]; } else { pos.append(c.gpa, a) catch return 2; } @@ -682,6 +715,9 @@ fn cmdFailed(c: Ctx, args: []const [:0]const u8) u8 { if (!allDigits(status)) die("STATUS must be a number, not '{s}'", .{status}); const logdir = fsx.openDir(fsx.cwd, pos.items[2], true) catch die("LOGDIR {s} is not a directory", .{pos.items[2]}); defer sys.close(logdir); + if (metrics_file) |p| if (p.len == 0) die("--metrics-file needs a path", .{}); + var kept: KeepStats = .{}; + defer updateMetrics(c, metrics_file, kept.logs, kept.bytes, false); const syslog_target = resolveSyslog(c, syslog_v); const logdir_s = joinable(pos.items[2]); if (std.mem.indexOfNone(u8, status, "0") == null) return 0; @@ -722,18 +758,24 @@ fn cmdFailed(c: Ctx, args: []const [:0]const u8) u8 { if (explicit == null) if (src_text) |t| if (scan.otherBuild(t, name)) { cli.out("No EasyBuild log to keep for {s}: {s} is another build's log.\n", .{ name, src_path.? }); const cmd_kept = keepCommandOutput(c, logdir, logdir_s, name, null); - emitFailedSyslog(c, syslog_target, status, name, logdir_s, false, cmd_kept, false, if (cmd_kept > 0) "kept" else "other-build"); + kept.logs += cmd_kept.logs; + kept.bytes += cmd_kept.bytes; + emitFailedSyslog(c, syslog_target, status, name, logdir_s, false, cmd_kept.logs, false, if (cmd_kept.logs > 0) "kept" else "other-build"); return 0; }; rewind(src.?.fd); const ebout = c.gpa.print("{s}.failed.ebout.log", .{name}) catch return 0; defer c.gpa.free(ebout); - const ebout_kept = keep(c, logdir, logdir_s, src.?.fd, src.?.stat, ebout, null); + const ebout_bytes = keep(c, logdir, logdir_s, src.?.fd, src.?.stat, ebout, null); + kept.add(ebout_bytes); + const ebout_kept = ebout_bytes != null; if (ebout_kept) cli.out("Failed build log: {s}/{s}\n", .{ logdir_s, ebout }); const cmd_kept = keepCommandOutput(c, logdir, logdir_s, name, src_text); + kept.logs += cmd_kept.logs; + kept.bytes += cmd_kept.bytes; // A meson test step prints only the last 100 lines of a failing test; // the full output is meson-logs/testlog.txt in the build directory. A @@ -753,13 +795,15 @@ fn cmdFailed(c: Ctx, args: []const [:0]const u8) u8 { rewind(tl.fd); const dest = c.gpa.print("{s}.failed.meson-testlog.txt", .{name}) catch break :blk; defer c.gpa.free(dest); - if (keep(c, logdir, logdir_s, tl.fd, tl.stat, dest, null)) { + const meson_bytes = keep(c, logdir, logdir_s, tl.fd, tl.stat, dest, null); + if (meson_bytes != null) { cli.out("Failed meson test log: {s}/{s}\n", .{ logdir_s, dest }); meson_kept = true; + kept.add(meson_bytes); } } else |_| {} }; - emitFailedSyslog(c, syslog_target, status, name, logdir_s, ebout_kept, cmd_kept, meson_kept, if (ebout_kept or cmd_kept > 0 or meson_kept) "kept" else "no-log"); + emitFailedSyslog(c, syslog_target, status, name, logdir_s, ebout_kept, cmd_kept.logs, meson_kept, if (ebout_kept or cmd_kept.logs > 0 or meson_kept) "kept" else "no-log"); return 0; } @@ -770,12 +814,12 @@ fn cmdFailed(c: Ctx, args: []const [:0]const u8) u8 { /// by this user. A stream is compared with EASYBUILD_TMPDIR with both /// resolved, as EasyBuild names the stream through EASYBUILD_TMPDIR as it was /// given. -fn keepCommandOutput(c: Ctx, logdir: fsx.fd_t, logdir_s: []const u8, name: []const u8, src_text: ?[]const u8) usize { - var kept_count: usize = 0; +fn keepCommandOutput(c: Ctx, logdir: fsx.fd_t, logdir_s: []const u8, name: []const u8, src_text: ?[]const u8) KeepStats { + var kept: KeepStats = .{}; var out_txt: ?[]const u8 = null; var stdout_text: ?[]u8 = null; defer if (stdout_text) |t| c.gpa.free(t); - const stdout_name = c.gpa.print("{s}.stdout.log", .{name}) catch return 0; + const stdout_name = c.gpa.print("{s}.stdout.log", .{name}) catch return kept; defer c.gpa.free(stdout_name); if (fsx.openRegular(logdir, stdout_name, true)) |f| { defer sys.close(f.fd); @@ -788,23 +832,23 @@ fn keepCommandOutput(c: Ctx, logdir: fsx.fd_t, logdir_s: []const u8, name: []con if (out_txt == null) if (src_text) |t| { out_txt = scan.failedCommandOutput(t); }; - const command_out = out_txt orelse return 0; + const command_out = out_txt orelse return kept; const tmpdir = c.getenv("EASYBUILD_TMPDIR") orelse ""; if (tmpdir.len == 0) { cli.out("Failed command output not kept: EASYBUILD_TMPDIR is not set.\n", .{}); - return 0; + return kept; } const root = command_out[0 .. command_out.len - "/out.txt".len]; var tb: [sys.path_max]u8 = undefined; const tmp_real: ?[]const u8 = fsx.realpath(fsx.cwd, tmpdir, &tb) catch null; const prefix: ?[]u8 = if (tmp_real) |t| - c.gpa.print("{s}/", .{if (t[t.len - 1] == '/') t[0 .. t.len - 1] else t}) catch return 0 + c.gpa.print("{s}/", .{if (t[t.len - 1] == '/') t[0 .. t.len - 1] else t}) catch return kept else null; defer if (prefix) |p| c.gpa.free(p); for ([_][]const u8{ "out", "err" }) |stream| { - const path = c.gpa.print("{s}/{s}.txt", .{ root, stream }) catch return kept_count; + const path = c.gpa.print("{s}/{s}.txt", .{ root, stream }) catch return kept; defer c.gpa.free(path); const f = fsx.openRegular(fsx.cwd, path, false) catch continue; defer sys.close(f.fd); @@ -818,14 +862,15 @@ fn keepCommandOutput(c: Ctx, logdir: fsx.fd_t, logdir_s: []const u8, name: []con cli.out("Failed command output not kept: {s} belongs to another user.\n", .{path}); continue; } - const dest = c.gpa.print("{s}.failed.command-{s}.txt", .{ name, stream }) catch return kept_count; + const dest = c.gpa.print("{s}.failed.command-{s}.txt", .{ name, stream }) catch return kept; defer c.gpa.free(dest); - if (keep(c, logdir, logdir_s, f.fd, f.stat, dest, 0o600)) { + const bytes = keep(c, logdir, logdir_s, f.fd, f.stat, dest, 0o600); + if (bytes != null) { cli.out("Failed command output: {s}/{s}\n", .{ logdir_s, dest }); - kept_count += 1; + kept.add(bytes); } } - return kept_count; + return kept; } fn parseDays(opt: []const u8, v: ?[]const u8) u64 { @@ -856,6 +901,12 @@ fn monotonicNs() i128 { return @as(i128, ts.sec) * std.time.ns_per_s + ts.nsec; } +fn realtimeSeconds() i64 { + var ts: linux.timespec = undefined; + _ = linux.clock_gettime(.REALTIME, &ts); + return ts.sec; +} + /// Open `root/.eb-keep-logs-rotate.lock`, created when missing; for reading /// when it cannot be opened for writing, by a user who may not write ROOT. /// A symbolic link or a file that is not a regular one is refused. @@ -1059,6 +1110,7 @@ fn cmdRotate(c: Ctx, args: []const [:0]const u8) u8 { var delete_v: ?[]const u8 = null; var lock_v: []const u8 = ""; var syslog_v: ?[]const u8 = null; + var metrics_file: ?[]const u8 = null; var have_compress = false; var have_delete = false; var i: usize = 0; @@ -1076,6 +1128,12 @@ fn cmdRotate(c: Ctx, args: []const [:0]const u8) u8 { syslog_v = args[i]; } else if (std.mem.startsWith(u8, a, "--syslog=")) { syslog_v = a["--syslog=".len..]; + } else if (std.mem.eql(u8, a, "--metrics-file")) { + if (i + 1 >= args.len) die("--metrics-file needs a path", .{}); + i += 1; + metrics_file = args[i]; + } else if (std.mem.startsWith(u8, a, "--metrics-file=")) { + metrics_file = a["--metrics-file=".len..]; } else if (std.mem.eql(u8, a, "--compress-after") or std.mem.eql(u8, a, "--delete-after")) { const v: ?[]const u8 = if (i + 1 < args.len) args[i + 1] else null; if (v == null) die("{s} needs a number of days", .{a}); @@ -1102,6 +1160,7 @@ fn cmdRotate(c: Ctx, args: []const [:0]const u8) u8 { } const root_p = root_s orelse die("rotate needs ROOT", .{}); if (root_p.len == 0) die("rotate needs ROOT", .{}); + if (metrics_file) |p| if (p.len == 0) die("--metrics-file needs a path", .{}); const root = fsx.openDir(fsx.cwd, root_p, true) catch die("ROOT {s} is not a directory", .{root_p}); defer sys.close(root); const compress_after = parseDays("--compress-after", if (have_compress) compress_v else null); @@ -1129,11 +1188,7 @@ fn cmdRotate(c: Ctx, args: []const [:0]const u8) u8 { // a lock manager, or NFS for a file open only for reading. else => warn("cannot lock {s}/{s}: {t}; rotating without the lock", .{ joinable(root_p), rotate_lock, err }), }; - const now: i64 = fixed_now orelse blk: { - var ts: linux.timespec = undefined; - _ = linux.clock_gettime(.REALTIME, &ts); - break :blk ts.sec; - }; + const now: i64 = fixed_now orelse realtimeSeconds(); var zone = localtime.load(c.gpa, c.getenv("TZ"), c.getenv("TZDIR")); defer zone.deinit(); @@ -1188,6 +1243,10 @@ fn cmdRotate(c: Ctx, args: []const [:0]const u8) u8 { const summary = arena.print("eb-keep-logs rotate: {d} file{s} compressed, {d} run{s} deleted under {s}", .{ compressed, plural(compressed), deleted, plural(deleted), root_p }) catch "eb-keep-logs rotate"; cli.out("{s}\n", .{summary}); emitRotateSyslog(c, syslog_target, root_p, compressed, deleted, summary); + // Run ages use the snapshot above. The metric chooses its completion time + // while holding the metrics lock, after all rotation work and event + // emission have completed. + updateMetrics(c, metrics_file, 0, 0, true); return 0; } @@ -1244,6 +1303,7 @@ test { _ = test_bundle; _ = lifecycle; _ = syslog; + _ = metrics; } test "a failed copy does not replace a log already kept" { diff --git a/src/metrics.zig b/src/metrics.zig new file mode 100644 index 0000000..9c80762 --- /dev/null +++ b/src/metrics.zig @@ -0,0 +1,202 @@ +//! Prometheus textfile metrics, updated atomically beside their destination. +const std = @import("std"); +const linux = std.os.linux; +const common = @import("zig_common"); +const fsx = common.fsx; +const sys = common.sys; + +const Allocator = std.mem.Allocator; + +const State = struct { + logs_kept: u64 = 0, + bytes_kept: u64 = 0, + last_prune: i64 = 0, +}; + +const logs_name = "eb_keep_logs_logs_kept_total"; +const bytes_name = "eb_keep_logs_bytes_kept_total"; +const prune_name = "eb_keep_logs_last_prune_timestamp_seconds"; + +fn parseSample(line: []const u8, comptime name: []const u8) !?[]const u8 { + if (!std.mem.startsWith(u8, line, name)) return null; + if (line.len <= name.len or line[name.len] != ' ') return error.InvalidMetrics; + const value = std.mem.trim(u8, line[name.len + 1 ..], " \t\r"); + if (value.len == 0) return error.InvalidMetrics; + return value; +} + +fn parse(text: []const u8) !State { + var state: State = .{}; + var have_logs = false; + var have_bytes = false; + var have_prune = false; + var lines = std.mem.splitScalar(u8, text, '\n'); + while (lines.next()) |line| { + if (try parseSample(line, logs_name)) |value| { + if (have_logs) return error.InvalidMetrics; + state.logs_kept = std.fmt.parseInt(u64, value, 10) catch return error.InvalidMetrics; + have_logs = true; + } else if (try parseSample(line, bytes_name)) |value| { + if (have_bytes) return error.InvalidMetrics; + state.bytes_kept = std.fmt.parseInt(u64, value, 10) catch return error.InvalidMetrics; + have_bytes = true; + } else if (try parseSample(line, prune_name)) |value| { + if (have_prune) return error.InvalidMetrics; + state.last_prune = std.fmt.parseInt(i64, value, 10) catch return error.InvalidMetrics; + if (state.last_prune < 0) return error.InvalidMetrics; + have_prune = true; + } + } + return state; +} + +fn render(gpa: Allocator, state: State) ![]u8 { + return gpa.print( + \\# HELP eb_keep_logs_logs_kept_total Log files kept by eb-keep-logs. + \\# TYPE eb_keep_logs_logs_kept_total counter + \\eb_keep_logs_logs_kept_total {d} + \\# HELP eb_keep_logs_bytes_kept_total Bytes written to kept log files by eb-keep-logs. + \\# TYPE eb_keep_logs_bytes_kept_total counter + \\eb_keep_logs_bytes_kept_total {d} + \\# HELP eb_keep_logs_last_prune_timestamp_seconds Unix timestamp of the last completed eb-keep-logs rotation. + \\# TYPE eb_keep_logs_last_prune_timestamp_seconds gauge + \\eb_keep_logs_last_prune_timestamp_seconds {d} + \\ + , .{ state.logs_kept, state.bytes_kept, state.last_prune }); +} + +fn lock(fd: fsx.fd_t) !void { + while (true) { + const rc = linux.flock(fd, std.posix.LOCK.EX); + if (linux.errno(rc) == .INTR) continue; + _ = try sys.result(rc); + return; + } +} + +fn realtimeSeconds() i64 { + var ts: linux.timespec = undefined; + _ = linux.clock_gettime(.REALTIME, &ts); + return ts.sec; +} + +fn keepLatestCompletion(state: *State, timestamp: i64) !void { + if (timestamp < 0) return error.InvalidTimestamp; + state.last_prune = @max(state.last_prune, timestamp); +} + +/// Add one invocation's known values to a Prometheus textfile. The file is +/// replaced by a synced temporary file in the same directory, under a +/// sidecar lock, so concurrent writers do not lose counter increments. A +/// completion time is chosen while holding that lock and never moves back. +pub fn update(gpa: Allocator, path: []const u8, logs_kept: u64, bytes_kept: u64, record_completion: bool) !void { + if (path.len == 0) return error.EmptyPath; + const base = std.fs.path.basename(path); + if (base.len == 0 or std.mem.eql(u8, base, ".") or std.mem.eql(u8, base, "..")) return error.EmptyPath; + const parent_path = std.fs.path.dirname(path) orelse "."; + const dir = try fsx.openDir(fsx.cwd, parent_path, true); + defer sys.close(dir); + + const lock_name = try gpa.print(".{s}.lock", .{base}); + defer gpa.free(lock_name); + const lock_fd = try sys.openat(dir, lock_name, .{ + .ACCMODE = .WRONLY, + .APPEND = true, + .CREAT = true, + .NOFOLLOW = true, + }, 0o600); + defer sys.close(lock_fd); + if (!(try fsx.fstat(lock_fd)).isReg()) return error.InvalidArgument; + try lock(lock_fd); + + var state: State = .{}; + var mode: u32 = 0o644; + if (fsx.openRegular(dir, base, false)) |existing| { + defer sys.close(existing.fd); + mode = existing.stat.perm(); + const text = try fsx.readAll(gpa, existing.fd, 1 << 20); + defer gpa.free(text); + state = try parse(text); + } else |err| switch (err) { + error.NotFound => {}, + else => |e| return e, + } + state.logs_kept = std.math.add(u64, state.logs_kept, logs_kept) catch return error.Overflow; + state.bytes_kept = std.math.add(u64, state.bytes_kept, bytes_kept) catch return error.Overflow; + if (record_completion) try keepLatestCompletion(&state, realtimeSeconds()); + const text = try render(gpa, state); + defer gpa.free(text); + + var random: [4]u8 = undefined; + _ = try sys.call(linux.getrandom, .{ &random, random.len, 0 }); + const tmp = try gpa.print(".{s}.{d}.{x:0>8}.tmp", .{ base, linux.getpid(), std.mem.readInt(u32, &random, .little) }); + defer gpa.free(tmp); + const fd = try sys.openat(dir, tmp, .{ + .ACCMODE = .WRONLY, + .CREAT = true, + .EXCL = true, + .NOFOLLOW = true, + }, @intCast(mode)); + var renamed = false; + defer { + sys.close(fd); + if (!renamed) sys.unlinkat(dir, tmp, 0) catch {}; + } + // openat applies the process umask even when replacing an existing file. + // Restore the exact previous mode (or 0644 for a new file), as the shell + // implementation's chmod does, before publishing the temporary file. + try sys.fchmod(fd, @intCast(mode)); + try sys.writeAll(fd, text); + try sys.fsync(fd); + try sys.renameat(dir, tmp, dir, base); + renamed = true; + try sys.fsync(dir); +} + +test "parse and render metrics" { + const gpa = std.testing.allocator; + const text = try render(gpa, .{ .logs_kept = 3, .bytes_kept = 57, .last_prune = 1_700_000_000 }); + defer gpa.free(text); + try std.testing.expectEqual(State{ + .logs_kept = 3, + .bytes_kept = 57, + .last_prune = 1_700_000_000, + }, try parse(text)); + try std.testing.expectError(error.InvalidMetrics, parse("eb_keep_logs_logs_kept_total NaN\n")); + try std.testing.expectError(error.InvalidMetrics, parse( + "eb_keep_logs_logs_kept_total 1\neb_keep_logs_logs_kept_total 2\n", + )); +} + +test "update preserves counters and replaces the whole file" { + const gpa = std.testing.allocator; + var cwd_buf: [sys.path_max]u8 = undefined; + const here = try fsx.realpath(fsx.cwd, ".", &cwd_buf); + const base = try std.fmt.allocPrint(gpa, "{s}/.zig-cache/eb-keep-logs-metrics-{d}", .{ here, linux.getpid() }); + defer gpa.free(base); + try common.atomic.makedirs(base); + defer fsx.deleteTree(gpa, fsx.cwd, base) catch {}; + const path = try std.fmt.allocPrint(gpa, "{s}/collector.prom", .{base}); + defer gpa.free(path); + + try update(gpa, path, 2, 19, false); + const created = try fsx.openRegular(fsx.cwd, path, false); + try std.testing.expectEqual(@as(u32, 0o644), created.stat.perm()); + try sys.fchmod(created.fd, 0o640); + sys.close(created.fd); + try update(gpa, path, 1, 5, false); + const file = try fsx.openRegular(fsx.cwd, path, false); + defer sys.close(file.fd); + try std.testing.expectEqual(@as(u32, 0o640), file.stat.perm()); + const text = try fsx.readAll(gpa, file.fd, 1 << 20); + defer gpa.free(text); + try std.testing.expectEqual(State{ .logs_kept = 3, .bytes_kept = 24, .last_prune = 0 }, try parse(text)); +} + +test "completion timestamps never move backwards" { + var state: State = .{ .last_prune = 1200 }; + try keepLatestCompletion(&state, 1234); + try keepLatestCompletion(&state, 1210); + try std.testing.expectEqual(@as(i64, 1234), state.last_prune); + try std.testing.expectError(error.InvalidTimestamp, keepLatestCompletion(&state, -1)); +} diff --git a/tests/test_metrics.sh b/tests/test_metrics.sh new file mode 100755 index 0000000..63bf3d7 --- /dev/null +++ b/tests/test_metrics.sh @@ -0,0 +1,133 @@ +#!/bin/bash +# Exercise the opt-in Prometheus textfile output. +set -u + +repo_root=$(cd "$(dirname "$0")/.." && pwd) +tool="${TOOL_BIN:-${repo_root}/eb-keep-logs}" +command -v promtool >/dev/null || { + echo "promtool is required for tests/test_metrics.sh" >&2 + exit 1 +} + +work=$(mktemp -d "${TMPDIR:-/tmp}/eb-keep-logs-metrics.XXXXXX") +trap 'rm -rf -- "${work}"' EXIT +mkdir -p "${work}/bin" "${work}/logs" "${work}/root" +printf 'hello\n' > "${work}/source.log" +cat > "${work}/bin/eb" </dev/null +[ ! -e "${metrics}" ] + +PATH="${work}/bin:${PATH}" "${tool}" failed --metrics-file "${metrics}" 1 First "${work}/logs" >/dev/null +PATH="${work}/bin:${PATH}" "${tool}" failed 1 Second "${work}/logs" --metrics-file="${metrics}" >/dev/null +promtool check metrics < "${metrics}" +grep -qx 'eb_keep_logs_logs_kept_total 2' "${metrics}" +grep -qx 'eb_keep_logs_bytes_kept_total 12' "${metrics}" +grep -qx 'eb_keep_logs_last_prune_timestamp_seconds 0' "${metrics}" + +# Concurrent writers serialize counter updates through the sidecar lock. +writers=() +for n in 1 2 3 4 5 6 7 8; do + PATH="${work}/bin:${PATH}" "${tool}" failed --metrics-file "${metrics}" 1 "Parallel-${n}" "${work}/logs" >/dev/null & + writers+=("$!") +done +for writer in "${writers[@]}"; do wait "${writer}"; done +promtool check metrics < "${metrics}" +grep -qx 'eb_keep_logs_logs_kept_total 10' "${metrics}" +grep -qx 'eb_keep_logs_bytes_kept_total 60' "${metrics}" + +# Restrictive umasks do not change a new file's documented mode or an +# existing destination's exact mode. +fresh="${work}/collector/fresh.prom" +(umask 077; PATH="${work}/bin:${PATH}" "${tool}" failed --metrics-file "${fresh}" 0 Fresh "${work}/logs" >/dev/null) +[ "$(stat -c %a "${fresh}")" = 644 ] +chmod 640 "${metrics}" +# Hold the sidecar lock across a second boundary. Completion time must be +# chosen after the lock is acquired, not before waiting for it. +metrics_lock="${work}/collector/.eb-keep-logs.prom.lock" +exec {held}>>"${metrics_lock}" +flock -n "${held}" +(umask 077; EB_KEEP_LOGS_NOW=1700000000 "${tool}" rotate "${work}/root" --compress-after 1 --metrics-file "${metrics}" >/dev/null) & +rotator=$! +sleep 2 +released=$(date +%s) +flock -u "${held}" +wait "${rotator}" +exec {held}>&- +finished=$(date +%s) +promtool check metrics < "${metrics}" +[ "$(stat -c %a "${metrics}")" = 640 ] +grep -qx 'eb_keep_logs_logs_kept_total 10' "${metrics}" +grep -qx 'eb_keep_logs_bytes_kept_total 60' "${metrics}" +pruned=$(awk '$1 == "eb_keep_logs_last_prune_timestamp_seconds" { print $2 }' "${metrics}") +[ "${pruned}" -ge "${released}" ] && [ "${pruned}" -le "${finished}" ] +[ "${pruned}" -ne 1700000000 ] + +# Concurrent rotations of different roots share the metrics lock. A later +# completion already present wins over out-of-order older completions. +future=$((finished + 3600)) +sed -i "s/^eb_keep_logs_last_prune_timestamp_seconds .*/eb_keep_logs_last_prune_timestamp_seconds ${future}/" "${metrics}" +mkdir -p "${work}/root-a" "${work}/root-b" +"${tool}" rotate "${work}/root-a" --compress-after 1 --metrics-file "${metrics}" >/dev/null & +rotate_a=$! +"${tool}" rotate "${work}/root-b" --compress-after 1 --metrics-file "${metrics}" >/dev/null & +rotate_b=$! +wait "${rotate_a}" "${rotate_b}" +promtool check metrics < "${metrics}" +grep -qx "eb_keep_logs_last_prune_timestamp_seconds ${future}" "${metrics}" + +# The shell implementation must not publish a temporary file when restoring +# its mode fails. The Zig implementation performs the equivalent fchmod +# before rename and its unit tests exercise that path's ordering. +if [ "$(head -c 2 "${tool}")" = '#!' ]; then + mkdir -p "${work}/no-chmod" + cat > "${work}/no-chmod/chmod" <<'EOF' +#!/bin/sh +exit 1 +EOF + /bin/chmod +x "${work}/no-chmod/chmod" + before=$(sha256sum "${metrics}") + mode_error=$(PATH="${work}/no-chmod:${PATH}" "${tool}" failed --metrics-file "${metrics}" 0 ModeFailure "${work}/logs" 2>&1 >/dev/null) + case "${mode_error}" in *"cannot set temporary file mode"*) ;; *) exit 1 ;; esac + [ "$(sha256sum "${metrics}")" = "${before}" ] + [ "$(find "${work}/collector" -maxdepth 1 -name '*.tmp' -print | wc -l)" -eq 0 ] +fi + +# A malformed prior file is preserved instead of resetting the counters. +printf 'eb_keep_logs_logs_kept_total NaN\n' > "${metrics}" +before=$(sha256sum "${metrics}") +PATH="${work}/bin:${PATH}" "${tool}" failed --metrics-file "${metrics}" 0 Third "${work}/logs" >/dev/null 2>&1 +[ "$(sha256sum "${metrics}")" = "${before}" ] + +# Both implementations accept the full u64 counter range and preserve the +# previous exposition when one more keep would overflow it. +cat > "${metrics}" <<'EOF' +# HELP eb_keep_logs_logs_kept_total Log files kept by eb-keep-logs. +# TYPE eb_keep_logs_logs_kept_total counter +eb_keep_logs_logs_kept_total 18446744073709551614 +# HELP eb_keep_logs_bytes_kept_total Bytes written to kept log files by eb-keep-logs. +# TYPE eb_keep_logs_bytes_kept_total counter +eb_keep_logs_bytes_kept_total 18446744073709551609 +# HELP eb_keep_logs_last_prune_timestamp_seconds Unix timestamp of the last completed eb-keep-logs rotation. +# TYPE eb_keep_logs_last_prune_timestamp_seconds gauge +eb_keep_logs_last_prune_timestamp_seconds 7 +EOF +PATH="${work}/bin:${PATH}" "${tool}" failed --metrics-file "${metrics}" 1 Boundary "${work}/logs" >/dev/null +promtool check metrics < "${metrics}" +grep -qx 'eb_keep_logs_logs_kept_total 18446744073709551615' "${metrics}" +grep -qx 'eb_keep_logs_bytes_kept_total 18446744073709551615' "${metrics}" +before=$(sha256sum "${metrics}") +overflow=$(PATH="${work}/bin:${PATH}" "${tool}" failed --metrics-file "${metrics}" 1 Overflow "${work}/logs" 2>&1 >/dev/null) +case "${overflow}" in *"counter overflow"*|*"Overflow"*) ;; *) exit 1 ;; esac +[ "$(sha256sum "${metrics}")" = "${before}" ] + +# No temporary output remains after any successful replacement. +[ "$(find "${work}/collector" -maxdepth 1 -name '*.tmp' -print | wc -l)" -eq 0 ] +echo "metrics tests passed"