diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d69cdd..4c4d08e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,13 @@ Notable changes to this stack. Format follows - The OpenTelemetry Collector runs the multi-arch `0.163.2` image again. The Dependabot bump to `0.163.2-386` pulled the 32-bit x86 build. `just lint` now refuses single-platform image tags. +- `run_scheduled.sh` stops its job when interrupted with Ctrl-C. Before, the + job kept running after the script had exited. Re-vendor the template to pick + this up. +- `just smoke` sends an admin password or OTLP token that contains `"` or `\` + as written. Before, curl changed it and the smoke run failed on auth. +- `just backup` prints the backup it just wrote, not a `pre-restore-*` + snapshot from an earlier restore. ### Removed diff --git a/justfile b/justfile index c5cb138..66b8e8f 100644 --- a/justfile +++ b/justfile @@ -110,6 +110,23 @@ lint: for bad in OTLP_AUTH_TOKEN=local-dev-token GRAFANA_ADMIN_PASSWORD=change-me GRAFANA_ROOT_URL=http://g.example GRAFANA_COOKIE_SECURE=false CF_ACCESS_AUD= ALERT_WEBHOOK_URL=; do \ ! env $good $bad just _expose-guards 2>/dev/null || { echo "error: exposure guards accepted $bad" >&2; exit 1; }; \ done + # run_scheduled.sh, interrupted mid-job, must take the job down with it, + # including what the job started (the stub's sleep stands in for a recipe's + # command under `just`). env resets INT, which bash ignores in a background job. + @d=$(mktemp -d) && mkdir "$d/templates" && cp templates/run_scheduled.sh "$d/templates/" && printf '#!/bin/sh\nsleep 30 & echo $! >"%s/pid"; wait\n' "$d" >"$d/job" && chmod +x "$d/job" || exit 1; \ + JUST_BIN="$d/job" env --default-signal=INT "$d/templates/run_scheduled.sh" job env >/dev/null 2>&1 & rs=$!; \ + for _ in $(seq 50); do [ -s "$d/pid" ] && break; sleep 0.1; done; kill -INT $rs; wait $rs; sleep 0.5; \ + p=$(cat "$d/pid" 2>/dev/null); rm -rf "$d"; [ -n "$p" ] || { echo "error: run_scheduled.sh never started its job" >&2; exit 1; }; \ + ! kill -0 "$p" 2>/dev/null || { kill "$p"; echo "error: run_scheduled.sh left its job running after SIGINT" >&2; exit 1; } + # The job reads /dev/null, not the caller's stdin: a job run by hand from a + # terminal would otherwise stop on its first read. + @d=$(mktemp -d) && mkdir "$d/templates" && cp templates/run_scheduled.sh "$d/templates/" && printf '#!/bin/sh\nread x && echo "read: $x"\n' >"$d/job" && chmod +x "$d/job" || exit 1; \ + out=$(echo caller-input | JUST_BIN="$d/job" "$d/templates/run_scheduled.sh" job env 2>&1); rm -rf "$d"; \ + case "$out" in *caller-input*) echo "error: run_scheduled.sh passed its stdin to the job" >&2; exit 1 ;; esac + # smoke.sh's cfg_quote must escape \ and " for curl's config parser. + @bash -c "$(sed -n '/^cfg_quote()/p' scripts/smoke.sh)"' && [ "$(cfg_quote "$1")" = "$2" ]' _ 'a"b\c' 'a\"b\\c' || { echo "error: cfg_quote no longer escapes \\ and \"" >&2; exit 1; } + # The backup listing names the new backup, not a pre-restore snapshot. + @d=$(mktemp -d) && touch "$d/monitoring-20261030-100000.tar.gz" "$d/pre-restore-20261030-110000.tar.gz" && just _newest-backup "$d" | grep -q 'monitoring-20261030-100000'; rc=$?; rm -rf "$d"; [ $rc -eq 0 ] || { echo "error: the backup listing does not name the new backup" >&2; exit 1; } # The rendered projects.yaml is skipped (its expr lines grow with every # project); its template is checked by rendering it into a scratch dir instead. docker run --rm --network none -v .:/code:ro {{yamllint}} yamllint -d '{extends: relaxed, rules: {line-length: {max: 120, allow-non-breakable-inline-mappings: true}}, ignore: [.git/, backups/, infra/.terraform/, config/grafana/alerting/projects.yaml]}' . @@ -176,7 +193,11 @@ _backup project dir: mkdir -p {{dir}} @-docker compose -p {{project}} unpause {{stateful}} >/dev/null 2>&1 @rc=0; img=$(just _image compose.yml otel-queue-init); m=$(just _mounts {{project}}); docker compose -p {{project}} pause {{stateful}} && docker run --rm --network none $m -v {{absolute_path(dir)}}:/backups "$img" sh -c 'set -o pipefail; umask 077 && tar cf - -C /data . | gzip -1 > /backups/monitoring-$(date +%Y%m%d-%H%M%S).tar.gz' || rc=$?; docker compose -p {{project}} unpause {{stateful}} || { echo "error: unpause failed; the stack is still paused" >&2; rc=1; }; exit $rc - @ls -lh {{dir}}/ | tail -1 + @just _newest-backup {{dir}} + +# pre-restore-* snapshots share the dir and sort after monitoring-* by name. +_newest-backup dir: + @ls -lh {{dir}}/monitoring-*.tar.gz | tail -1 # Restore a backup tarball into the volumes (stops the stack; wipes current state). restore file: (_restore core_project file "backups") diff --git a/scripts/smoke.sh b/scripts/smoke.sh index bece49d..f895d0a 100755 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -9,8 +9,10 @@ project="${SMOKE_PROJECT:?}" command -v jq >/dev/null || { echo "error: jq is required" >&2; exit 1; } die() { echo "error: $*" >&2; exit 1; } +# curl's config parser unescapes \ and " inside a quoted value, so escape both. +cfg_quote() { local v="${1//\\/\\\\}"; printf '%s' "${v//\"/\\\"}"; } # Admin basic auth through curl's stdin config: argv is world-readable in ps. -gf() { printf 'user = "admin:%s"\n' "$GRAFANA_ADMIN_PASSWORD" | curl -sf -K - "$@"; } +gf() { printf 'user = "admin:%s"\n' "$(cfg_quote "$GRAFANA_ADMIN_PASSWORD")" | curl -sf -K - "$@"; } promq() { gf -G --data-urlencode "query=$1" "$url/api/datasources/proxy/uid/prometheus/api/v1/query"; } poll() { local n=0; until "$@"; do n=$((n + 1)); [[ $n -lt ${POLL_MAX:-60} ]] || return 1; sleep "${POLL_SLEEP:-1}"; done; } @@ -78,7 +80,7 @@ res='{"attributes":[{"key":"service.name","value":{"stringValue":"smoke"}},{"key otlp() { docker compose -p "$project" exec -T grafana sh -c \ 'curl -sf -K - -o /dev/null -H "Content-Type: application/json" --data-binary "$1" "http://otel-collector:4318/v1/$2"' \ - _ "$2" "$1" <<<"header = \"Authorization: Bearer $OTLP_AUTH_TOKEN\"" + _ "$2" "$1" <<<"header = \"Authorization: Bearer $(cfg_quote "$OTLP_AUTH_TOKEN")\"" } otlp metrics '{"resourceMetrics":[{"resource":'"$res"',"scopeMetrics":[{"metrics":[{"name":"smoke_up","gauge":{"dataPoints":[{"asInt":"1","timeUnixNano":"'"$ts"'"}]}}]}]}]}' \ || die "the collector refused an OTLP metric with the .env token" diff --git a/templates/run_scheduled.sh b/templates/run_scheduled.sh index eda43b9..3812c1e 100755 --- a/templates/run_scheduled.sh +++ b/templates/run_scheduled.sh @@ -58,10 +58,13 @@ ping_status() { # A killed job must still report: systemd's TimeoutStartSec TERMs the whole cgroup, and # without this trap a hung job would send neither ping. The job runs in the background -# so `wait` can be interrupted; the child receives the same TERM from systemd. +# so `wait` can be interrupted. systemd TERMs the job too, but an INT (Ctrl-C) reaches +# only this script, so the trap TERMs the job's process group: the job is usually +# `just`, which does not pass the signal on to what its recipe started. # shellcheck disable=SC2329 # invoked via the TERM/INT traps below on_terminate() { local sig="$1" + [[ -z "${job_pid:-}" ]] || kill -TERM -- "-$job_pid" 2>/dev/null echo "run_scheduled: received SIG${sig}; job killed (likely a systemd timeout)" >>"$output_file" cat "$output_file" if [[ -n "$ping_url" ]]; then @@ -74,8 +77,14 @@ trap 'on_terminate TERM' TERM trap 'on_terminate INT' INT status=0 -"${command[@]}" >"$output_file" 2>&1 & -wait $! || status=$? +# Job control on for the fork only, so the job gets its own process group. +# With it on, bash no longer gives a background job /dev/null as stdin, so a +# job run by hand would stop on a terminal read: pass /dev/null explicitly. +set -m +"${command[@]}" >"$output_file" 2>&1 & +job_pid=$! +set +m +wait "$job_pid" || status=$? cat "$output_file" [[ -z "$ping_url" ]] || ping_status "$status"