fix: stop interrupted scheduled jobs, escape smoke credentials, show the new backup

CHANGELOG.md

@@ -19,6 +19,13 @@ Notable changes to this stack. Format follows
 - The OpenTelemetry Collector runs the multi-arch `0.163.2` image again. The
   Dependabot bump to `0.163.2-386` pulled the 32-bit x86 build. `just lint` now
   refuses single-platform image tags.
+- `run_scheduled.sh` stops its job when interrupted with Ctrl-C. Before, the
+  job kept running after the script had exited. Re-vendor the template to pick
+  this up.
+- `just smoke` sends an admin password or OTLP token that contains `"` or `\`
+  as written. Before, curl changed it and the smoke run failed on auth.
+- `just backup` prints the backup it just wrote, not a `pre-restore-*`
+  snapshot from an earlier restore.
 
 ### Removed
 

justfile

@@ -110,6 +110,23 @@ lint:
       for bad in OTLP_AUTH_TOKEN=local-dev-token GRAFANA_ADMIN_PASSWORD=change-me GRAFANA_ROOT_URL=http://g.example GRAFANA_COOKIE_SECURE=false CF_ACCESS_AUD= ALERT_WEBHOOK_URL=; do \
         ! env $good $bad just _expose-guards 2>/dev/null || { echo "error: exposure guards accepted $bad" >&2; exit 1; }; \
       done
+    # run_scheduled.sh, interrupted mid-job, must take the job down with it,
+    # including what the job started (the stub's sleep stands in for a recipe's
+    # command under `just`). env resets INT, which bash ignores in a background job.
+    @d=$(mktemp -d) && mkdir "$d/templates" && cp templates/run_scheduled.sh "$d/templates/" && printf '#!/bin/sh\nsleep 30 & echo $! >"%s/pid"; wait\n' "$d" >"$d/job" && chmod +x "$d/job" || exit 1; \
+      JUST_BIN="$d/job" env --default-signal=INT "$d/templates/run_scheduled.sh" job env >/dev/null 2>&1 & rs=$!; \
+      for _ in $(seq 50); do [ -s "$d/pid" ] && break; sleep 0.1; done; kill -INT $rs; wait $rs; sleep 0.5; \
+      p=$(cat "$d/pid" 2>/dev/null); rm -rf "$d"; [ -n "$p" ] || { echo "error: run_scheduled.sh never started its job" >&2; exit 1; }; \
+      ! kill -0 "$p" 2>/dev/null || { kill "$p"; echo "error: run_scheduled.sh left its job running after SIGINT" >&2; exit 1; }
+    # The job reads /dev/null, not the caller's stdin: a job run by hand from a
+    # terminal would otherwise stop on its first read.
+    @d=$(mktemp -d) && mkdir "$d/templates" && cp templates/run_scheduled.sh "$d/templates/" && printf '#!/bin/sh\nread x && echo "read: $x"\n' >"$d/job" && chmod +x "$d/job" || exit 1; \
+      out=$(echo caller-input | JUST_BIN="$d/job" "$d/templates/run_scheduled.sh" job env 2>&1); rm -rf "$d"; \
+      case "$out" in *caller-input*) echo "error: run_scheduled.sh passed its stdin to the job" >&2; exit 1 ;; esac
+    # smoke.sh's cfg_quote must escape \ and " for curl's config parser.
+    @bash -c "$(sed -n '/^cfg_quote()/p' scripts/smoke.sh)"' && [ "$(cfg_quote "$1")" = "$2" ]' _ 'a"b\c' 'a\"b\\c' || { echo "error: cfg_quote no longer escapes \\ and \"" >&2; exit 1; }
+    # The backup listing names the new backup, not a pre-restore snapshot.
+    @d=$(mktemp -d) && touch "$d/monitoring-20261030-100000.tar.gz" "$d/pre-restore-20261030-110000.tar.gz" && just _newest-backup "$d" | grep -q 'monitoring-20261030-100000'; rc=$?; rm -rf "$d"; [ $rc -eq 0 ] || { echo "error: the backup listing does not name the new backup" >&2; exit 1; }
     # The rendered projects.yaml is skipped (its expr lines grow with every
     # project); its template is checked by rendering it into a scratch dir instead.
     docker run --rm --network none -v .:/code:ro {{yamllint}} yamllint -d '{extends: relaxed, rules: {line-length: {max: 120, allow-non-breakable-inline-mappings: true}}, ignore: [.git/, backups/, infra/.terraform/, config/grafana/alerting/projects.yaml]}' .
@@ -176,7 +193,11 @@ _backup project dir:
     mkdir -p {{dir}}
     @-docker compose -p {{project}} unpause {{stateful}} >/dev/null 2>&1
     @rc=0; img=$(just _image compose.yml otel-queue-init); m=$(just _mounts {{project}}); docker compose -p {{project}} pause {{stateful}} && docker run --rm --network none $m -v {{absolute_path(dir)}}:/backups "$img" sh -c 'set -o pipefail; umask 077 && tar cf - -C /data . | gzip -1 > /backups/monitoring-$(date +%Y%m%d-%H%M%S).tar.gz' || rc=$?; docker compose -p {{project}} unpause {{stateful}} || { echo "error: unpause failed; the stack is still paused" >&2; rc=1; }; exit $rc
-    @ls -lh {{dir}}/ | tail -1
+    @just _newest-backup {{dir}}
+
+# pre-restore-* snapshots share the dir and sort after monitoring-* by name.
+_newest-backup dir:
+    @ls -lh {{dir}}/monitoring-*.tar.gz | tail -1
 
 # Restore a backup tarball into the volumes (stops the stack; wipes current state).
 restore file: (_restore core_project file "backups")

scripts/smoke.sh

@@ -9,8 +9,10 @@ project="${SMOKE_PROJECT:?}"
 command -v jq >/dev/null || { echo "error: jq is required" >&2; exit 1; }
 
 die() { echo "error: $*" >&2; exit 1; }
+# curl's config parser unescapes \ and " inside a quoted value, so escape both.
+cfg_quote() { local v="${1//\\/\\\\}"; printf '%s' "${v//\"/\\\"}"; }
 # Admin basic auth through curl's stdin config: argv is world-readable in ps.
-gf() { printf 'user = "admin:%s"\n' "$GRAFANA_ADMIN_PASSWORD" | curl -sf -K - "$@"; }
+gf() { printf 'user = "admin:%s"\n' "$(cfg_quote "$GRAFANA_ADMIN_PASSWORD")" | curl -sf -K - "$@"; }
 promq() { gf -G --data-urlencode "query=$1" "$url/api/datasources/proxy/uid/prometheus/api/v1/query"; }
 poll() { local n=0; until "$@"; do n=$((n + 1)); [[ $n -lt ${POLL_MAX:-60} ]] || return 1; sleep "${POLL_SLEEP:-1}"; done; }
 
@@ -78,7 +80,7 @@ res='{"attributes":[{"key":"service.name","value":{"stringValue":"smoke"}},{"key
 otlp() {
     docker compose -p "$project" exec -T grafana sh -c \
         'curl -sf -K - -o /dev/null -H "Content-Type: application/json" --data-binary "$1" "http://otel-collector:4318/v1/$2"' \
-        _ "$2" "$1" <<<"header = \"Authorization: Bearer $OTLP_AUTH_TOKEN\""
+        _ "$2" "$1" <<<"header = \"Authorization: Bearer $(cfg_quote "$OTLP_AUTH_TOKEN")\""
 }
 otlp metrics '{"resourceMetrics":[{"resource":'"$res"',"scopeMetrics":[{"metrics":[{"name":"smoke_up","gauge":{"dataPoints":[{"asInt":"1","timeUnixNano":"'"$ts"'"}]}}]}]}]}' \
     || die "the collector refused an OTLP metric with the .env token"

templates/run_scheduled.sh

@@ -58,10 +58,13 @@ ping_status() {
 
 # A killed job must still report: systemd's TimeoutStartSec TERMs the whole cgroup, and
 # without this trap a hung job would send neither ping. The job runs in the background
-# so `wait` can be interrupted; the child receives the same TERM from systemd.
+# so `wait` can be interrupted. systemd TERMs the job too, but an INT (Ctrl-C) reaches
+# only this script, so the trap TERMs the job's process group: the job is usually
+# `just`, which does not pass the signal on to what its recipe started.
 # shellcheck disable=SC2329  # invoked via the TERM/INT traps below
 on_terminate() {
     local sig="$1"
+    [[ -z "${job_pid:-}" ]] || kill -TERM -- "-$job_pid" 2>/dev/null
     echo "run_scheduled: received SIG${sig}; job killed (likely a systemd timeout)" >>"$output_file"
     cat "$output_file"
     if [[ -n "$ping_url" ]]; then
@@ -74,8 +77,14 @@ trap 'on_terminate TERM' TERM
 trap 'on_terminate INT' INT
 
 status=0
-"${command[@]}" >"$output_file" 2>&1 &
-wait $! || status=$?
+# Job control on for the fork only, so the job gets its own process group.
+# With it on, bash no longer gives a background job /dev/null as stdin, so a
+# job run by hand would stop on a terminal read: pass /dev/null explicitly.
+set -m
+"${command[@]}" </dev/null >>"$output_file" 2>&1 &
+job_pid=$!
+set +m
+wait "$job_pid" || status=$?
 cat "$output_file"
 
 [[ -z "$ping_url" ]] || ping_status "$status"